diff --git a/shared/yeast-macros/src/lib.rs b/shared/yeast-macros/src/lib.rs index 0df96c91d26..82b0e3e7b40 100644 --- a/shared/yeast-macros/src/lib.rs +++ b/shared/yeast-macros/src/lib.rs @@ -47,8 +47,35 @@ pub fn query(input: TokenStream) -> TokenStream { /// `Option`, iterator chains) splice /// their elements /// field: {expr} - extend a named field with `{expr}`'s ids +/// field: (kind ...)? - set the field only if every `#{expr}` +/// beneath it has a value (see below) /// ``` /// +/// # Optional fields +/// +/// A `?` on a field's value makes that field fallible: if a `#{expr}` +/// anywhere beneath it interpolates an absent value — an `Option` that is +/// `None` — the subtree is abandoned and the field is left unset. This +/// replaces the surrounding `Option::map` that would otherwise be needed: +/// +/// ```text +/// (break_expr label: {lbl.map(|l| tree!((identifier #{l})))}) // before +/// (break_expr label: (identifier #{lbl})?) // after +/// ``` +/// +/// The value may be nested arbitrarily deeply, and a nested `?` catches +/// first, so an inner absent value need not discard the outer node: +/// +/// ```text +/// (parameter pattern: (name_pattern identifier: (identifier #{name}))? type: {ty}) +/// ``` +/// +/// Only `#{expr}` propagates absence. A `{expr}` splice is unaffected, since +/// yielding no ids already leaves a field unset, and `?` is rejected on one. +/// Outside a `?`, interpolating an `Option` with `#{expr}` remains a compile +/// error, so the choice between "unset the field" and "unwrap it" stays +/// explicit. +/// /// Can be called with an explicit context or using the implicit context /// from an enclosing `rule!`: /// diff --git a/shared/yeast-macros/src/parse.rs b/shared/yeast-macros/src/parse.rs index 55ada358849..40f9dbce61c 100644 --- a/shared/yeast-macros/src/parse.rs +++ b/shared/yeast-macros/src/parse.rs @@ -1,10 +1,44 @@ use proc_macro2::{Delimiter, Ident, Literal, Span, TokenStream, TokenTree}; use quote::quote; use std::iter::Peekable; +use std::sync::atomic::{AtomicUsize, Ordering}; +use syn::Lifetime; type Tokens = Peekable; type Result = std::result::Result; +/// Mints the block label a fallible field breaks out of. Labels must be unique +/// along a nesting chain, since a `?` nested inside another `?` has to break out +/// of the inner field only. +fn fresh_fallible_label() -> Lifetime { + static COUNTER: AtomicUsize = AtomicUsize::new(0); + let n = COUNTER.fetch_add(1, Ordering::Relaxed); + Lifetime::new(&format!("'__yeast_field_{n}"), Span::call_site()) +} + +/// Rejects a `?` in a position where there is no field for it to leave unset. +/// +/// The advice depends on what precedes it: on a `{...}` splice a `?` is +/// redundant, and anywhere else it belongs on a field's value. Other +/// quantifiers need no handling — `*` and `+` have never been valid in a +/// template, and the existing check for stray tokens already rejects them. +fn reject_stray_optional(tokens: &mut Tokens, after_splice: bool) -> Result<()> { + let Some(TokenTree::Punct(p)) = tokens.peek() else { + return Ok(()); + }; + if p.as_char() != '?' { + return Ok(()); + } + let msg = if after_splice { + "`?` is not valid on a `{...}` splice; a splice that yields no value \ + already leaves its field unset" + } else { + "`?` is only valid on the value of a named field, as in \ + `label: (identifier #{lbl})?`" + }; + Err(syn::Error::new_spanned(p.clone(), msg)) +} + // --------------------------------------------------------------------------- // Query parsing // --------------------------------------------------------------------------- @@ -318,8 +352,9 @@ pub fn parse_tree_top(input: TokenStream) -> Result { let mut tokens = input.into_iter().peekable(); let ctx = parse_ctx_or_implicit(&mut tokens); - let first = parse_direct_node(&mut tokens, &ctx)?; + let first = parse_direct_node(&mut tokens, &ctx, None)?; + reject_stray_optional(&mut tokens, false)?; if let Some(tok) = tokens.next() { return Err(syn::Error::new_spanned( tok, @@ -352,7 +387,15 @@ pub fn parse_trees_top(input: TokenStream) -> Result { /// Parse a single node template and generate code that returns an `Id`. /// Handles: `(kind fields... children...)` and `{expr}`. -fn parse_direct_node(tokens: &mut Tokens, ctx: &Ident) -> Result { +/// +/// `scope` is the enclosing fallible field's label, if any: inside one, a +/// `#{expr}` that interpolates an absent value breaks out to it, leaving that +/// field unset. See [`parse_direct_node_inner`]. +fn parse_direct_node( + tokens: &mut Tokens, + ctx: &Ident, + scope: Option<&Lifetime>, +) -> Result { match tokens.peek() { Some(TokenTree::Group(g)) if g.delimiter() == Delimiter::Brace => { let group = expect_group(tokens, Delimiter::Brace)?; @@ -362,7 +405,7 @@ fn parse_direct_node(tokens: &mut Tokens, ctx: &Ident) -> Result { Some(TokenTree::Group(g)) if g.delimiter() == Delimiter::Parenthesis => { let group = expect_group(tokens, Delimiter::Parenthesis)?; let mut inner = group.stream().into_iter().peekable(); - parse_direct_node_inner(&mut inner, ctx) + parse_direct_node_inner(&mut inner, ctx, scope) } Some(tok) => Err(syn::Error::new_spanned( tok.clone(), @@ -377,7 +420,11 @@ fn parse_direct_node(tokens: &mut Tokens, ctx: &Ident) -> Result { /// Parse the inside of a parenthesized node: `kind fields... children...` /// or `kind "literal"` or `kind $fresh`. -fn parse_direct_node_inner(tokens: &mut Tokens, ctx: &Ident) -> Result { +fn parse_direct_node_inner( + tokens: &mut Tokens, + ctx: &Ident, + scope: Option<&Lifetime>, +) -> Result { let kind = expect_ident(tokens, "expected node kind")?; let kind_str = kind.to_string(); @@ -392,6 +439,28 @@ fn parse_direct_node_inner(tokens: &mut Tokens, ctx: &Ident) -> Result Result = Vec::new(); @@ -446,7 +516,47 @@ fn parse_direct_node_inner(tokens: &mut Tokens, ctx: &Ident) -> Result = #label: { + ::std::option::Option::Some(#value) + }; + }); + field_args.push(quote! { + if let ::std::option::Option::Some(__id) = #temp { + __fields.push((#field_str, vec![__id])); + } + }); + } else { + // No `?` of its own, so failures beneath it belong to whichever + // fallible field encloses this one, if any. + let value = parse_direct_node_inner(&mut inner, ctx, scope)?; + stmts.push(quote! { let #temp: yeast::Id = #value; }); + field_args.push(quote! { __fields.push((#field_str, vec![#temp])); }); + } + continue; + } + + // Neither form matched; delegate for a consistent error message. + let value = parse_direct_node(tokens, ctx, scope)?; stmts.push(quote! { let #temp: yeast::Id = #value; }); field_args.push(quote! { __fields.push((#field_str, vec![#temp])); }); } @@ -486,7 +596,8 @@ fn parse_direct_list(tokens: &mut Tokens, ctx: &Ident) -> Result Result bool { matches!(tokens.peek(), Some(TokenTree::Group(g)) if g.delimiter() == delim) } +fn peek_is_punct(tokens: &mut Tokens, ch: char) -> bool { + matches!(tokens.peek(), Some(TokenTree::Punct(p)) if p.as_char() == ch) +} + fn peek_is_repetition(tokens: &mut Tokens) -> bool { matches!(tokens.peek(), Some(TokenTree::Punct(p)) if matches!(p.as_char(), '*' | '+' | '?')) } diff --git a/shared/yeast/doc/yeast.md b/shared/yeast/doc/yeast.md index 90edb510c1a..be3bc913c70 100644 --- a/shared/yeast/doc/yeast.md +++ b/shared/yeast/doc/yeast.md @@ -235,6 +235,48 @@ yeast::trees!(ctx, (identifier #{name}) // an identifier from a Rust variable ``` +### Optional fields (`?`) + +A `?` on a field's value makes that field fallible. If a `#{expr}` anywhere +beneath it interpolates an absent value — an `Option` that is `None` — the +subtree is abandoned and the field is left unset: + +```rust +rule!((breakStmt label: _? @@lbl) => (break_expr label: (identifier #{lbl})?)) +``` + +Here an optional capture is being wrapped in a leaf, which without `?` needs +an `Option::map` to build the leaf only in the `Some` case: + +```rust +// Equivalent, but the intent is buried in the closure: +(break_expr label: {lbl.map(|l| tree!((identifier #{l})))}) +``` + +The marker mirrors the query language, where a quantifier likewise follows the +value it applies to (`label: _? @@lbl`). Note that the schema puts it on the +other side of the colon — `external_name?: identifier` — because it is +*declaring* a field's cardinality rather than supplying a value for it. + +Absence propagates outwards through as many levels as necessary, and a nested +`?` catches first, so an inner absent value need not discard the outer node: + +```rust +// If `name` is absent, `pattern` is left unset — but `type` is still set. +(parameter pattern: (name_pattern identifier: (identifier #{name}))? type: {ty}) +``` + +Only `#{expr}` propagates absence, because it supplies a node's *content*: with +no value there is no leaf to build. A `{expr}` splice supplies *children*, where +yielding nothing already leaves the field unset (see below), so `?` is rejected +on one. Repetition has no marker at all, in templates or elsewhere: how many +children a `{expr}` contributes is a property of the expression rather than of +the syntax. + +Outside a `?`, interpolating an `Option` with `#{expr}` remains a compile error. +That is deliberate: it keeps the choice between "leave the field unset" and +"unwrap it" explicit at every interpolation. + ### Fresh identifiers `(kind $name)` creates a leaf node with an auto-generated unique name. All @@ -279,6 +321,11 @@ yeast::trees!(ctx, ) ``` +Because an `Option` splices as zero or one id, `field: {opt}` already +leaves the field unset when `opt` is `None`. Use [`?`](#optional-fields-) +instead when the optional value has to be *wrapped* in a node first, so that +there is nothing to wrap when it is absent. + The contents of `{…}` are treated as a Rust block, so multi-statement expressions (with `let` bindings) work too: diff --git a/shared/yeast/src/lib.rs b/shared/yeast/src/lib.rs index 14a0ab05576..cf06d2d2c17 100644 --- a/shared/yeast/src/lib.rs +++ b/shared/yeast/src/lib.rs @@ -117,6 +117,11 @@ where /// All standard primitive and string types implement [`YeastDisplay`] via /// the [`impl_yeast_display_via_display`] macro below. Coherence prevents a /// blanket `impl`, so additional types must be added explicitly. +#[diagnostic::on_unimplemented( + message = "`{Self}` cannot be interpolated with `#{{...}}`", + note = "if the value is optional, mark the enclosing field's value with `?` to leave \ + that field unset when it is absent, as in `label: (identifier #{{lbl}})?`" +)] pub trait YeastDisplay { fn yeast_to_string(&self, ast: &Ast) -> String; } @@ -183,6 +188,81 @@ impl YeastSourceRange for &T { } } +/// Normalizes a `#{expr}` interpolation to an optional value, so that a +/// fallible field — `field: (kind #{expr})?` — can tell "there is a value to +/// interpolate" apart from "there is none, so leave the field unset". +/// +/// Implemented for every [`YeastDisplay`] type, which always yields a value, +/// and for `Option` of the same, which yields one only when it is `Some`. +/// +/// The implementations are enumerated rather than blanket: a blanket +/// `impl` would overlap with the `Option` impl, since +/// coherence cannot rule out a future `impl YeastDisplay for Option`. +/// [`YeastDisplay`] itself is enumerated for the same reason. +/// +/// Note that this is used *only* inside a fallible field. Elsewhere `#{expr}` +/// still goes directly through [`YeastDisplay`], so interpolating an `Option` +/// without a `?` remains a compile error rather than silently dropping a node. +pub trait MaybeYeastValue { + /// The interpolated value's type, which knows how to render itself. + type Value: YeastDisplay + YeastSourceRange + ?Sized; + + /// Returns the value to interpolate, or `None` to leave the field unset. + fn maybe_yeast_value(&self) -> Option<&Self::Value>; +} + +macro_rules! impl_maybe_yeast_value { + ($($t:ty),* $(,)?) => { + $( + impl MaybeYeastValue for $t { + type Value = $t; + fn maybe_yeast_value(&self) -> Option<&$t> { + Some(self) + } + } + + impl MaybeYeastValue for Option<$t> { + type Value = $t; + fn maybe_yeast_value(&self) -> Option<&$t> { + self.as_ref() + } + } + )* + }; +} + +impl_maybe_yeast_value! { + Id, + i8, i16, i32, i64, i128, isize, + u8, u16, u32, u64, u128, usize, + f32, f64, + bool, char, + String, +} + +// `str` is unsized, so it has no `Option` counterpart; `Option<&str>` is +// covered by the reference impls below. +impl MaybeYeastValue for str { + type Value = str; + fn maybe_yeast_value(&self) -> Option<&str> { + Some(self) + } +} + +impl MaybeYeastValue for &T { + type Value = T::Value; + fn maybe_yeast_value(&self) -> Option<&T::Value> { + (**self).maybe_yeast_value() + } +} + +impl MaybeYeastValue for Option<&T> { + type Value = T::Value; + fn maybe_yeast_value(&self) -> Option<&T::Value> { + (*self).and_then(MaybeYeastValue::maybe_yeast_value) + } +} + #[derive(Debug)] pub struct AstCursor<'a> { ast: &'a Ast, diff --git a/shared/yeast/tests/test.rs b/shared/yeast/tests/test.rs index 756e219bd89..bdc17c7593d 100644 --- a/shared/yeast/tests/test.rs +++ b/shared/yeast/tests/test.rs @@ -674,6 +674,117 @@ fn test_tree_builder() { ); } +/// Builds `(assignment left: … right: (integer #{value})?)`, where a `None` +/// `value` leaves `right` unset rather than producing an `integer` with no +/// content. +fn build_optional_right(ast: &mut Ast, value: Option) -> (yeast::Id, yeast::Id) { + let captures = yeast::captures::Captures::new(); + let fresh = yeast::tree_builder::FreshScope::new(); + let mut user_ctx = (); + let mut ctx = yeast::build::BuildCtx::new(ast, &captures, &fresh, &mut user_ctx); + let left = yeast::tree!(ctx, (identifier "x")); + let root = yeast::tree!(ctx, + (assignment + left: {left} + right: (integer #{value})? + ) + ); + (root, left) +} + +#[test] +fn test_optional_field_is_set_when_the_value_is_present() { + let runner: Runner = Runner::new(tree_sitter_ruby::LANGUAGE.into(), &[]); + let mut ast = runner.run("x = 1").unwrap(); + + // Any node will do as the interpolated value; `#{…}` renders its source text. + let mut cursor = AstCursor::new(&ast); + cursor.goto_first_child(); + let some = cursor.node_id(); + + let (root, _) = build_optional_right(&mut ast, Some(some)); + assert_dump_eq( + &dump_ast(&ast, root, "x = 1"), + r#" + assignment + left: identifier "x" + right: integer "x = 1" + "#, + ); +} + +#[test] +fn test_optional_field_is_unset_when_the_value_is_absent() { + let runner: Runner = Runner::new(tree_sitter_ruby::LANGUAGE.into(), &[]); + let mut ast = runner.run("x = 1").unwrap(); + + let (root, _) = build_optional_right(&mut ast, None); + assert_dump_eq( + &dump_ast(&ast, root, "x = 1"), + r#" + assignment + left: identifier "x" + "#, + ); +} + +#[test] +fn test_optional_field_propagates_through_nested_nodes() { + let runner: Runner = Runner::new(tree_sitter_ruby::LANGUAGE.into(), &[]); + let mut ast = runner.run("x = 1").unwrap(); + + let captures = yeast::captures::Captures::new(); + let fresh = yeast::tree_builder::FreshScope::new(); + let mut user_ctx = (); + let mut ctx = yeast::build::BuildCtx::new(&mut ast, &captures, &fresh, &mut user_ctx); + + // The absent value sits two levels below the `?`, so the whole + // `left_assignment_list` subtree is abandoned along with it. + let absent: Option = None; + let right = yeast::tree!(ctx, (integer "1")); + let root = yeast::tree!(ctx, + (assignment + left: (left_assignment_list child: (identifier #{absent}))? + right: {right} + ) + ); + + assert_dump_eq( + &dump_ast(&ast, root, "x = 1"), + r#" + assignment + right: integer "1" + "#, + ); +} + +#[test] +fn test_innermost_optional_field_catches_first() { + let runner: Runner = Runner::new(tree_sitter_ruby::LANGUAGE.into(), &[]); + let mut ast = runner.run("x = 1").unwrap(); + + let captures = yeast::captures::Captures::new(); + let fresh = yeast::tree_builder::FreshScope::new(); + let mut user_ctx = (); + let mut ctx = yeast::build::BuildCtx::new(&mut ast, &captures, &fresh, &mut user_ctx); + + // The inner `?` catches, so only `child` is dropped; `left` survives. + let absent: Option = None; + let root = yeast::tree!(ctx, + (assignment + left: (left_assignment_list child: (identifier #{absent})?)? + ) + ); + + assert_dump_eq( + &dump_ast(&ast, root, "x = 1"), + r#" + assignment + left: left_assignment_list + "#, + ); +} + // ---- Rule tests ---- // These rules use field names from node-types.yml, which extends the diff --git a/unified/extractor/src/languages/swift/swift.rs b/unified/extractor/src/languages/swift/swift.rs index a3b05cbdb1a..bb421e77698 100644 --- a/unified/extractor/src/languages/swift/swift.rs +++ b/unified/extractor/src/languages/swift/swift.rs @@ -412,7 +412,7 @@ fn translation_rules() -> Vec> { rule!( (enumCaseParameter firstName: _? @@name type: @ty) => - (parameter pattern: {name.map(|name| tree!((name_pattern identifier: (identifier #{name}))))} type: {ty}) + (parameter pattern: (name_pattern identifier: (identifier #{name}))? type: {ty}) ), // An enum element with associated values (`case circle(radius: Double)`) // becomes a nested `class_like_declaration` whose constructor carries the @@ -508,9 +508,9 @@ fn translation_rules() -> Vec> { // key; unlabelled elements have no key. rule!((tuplePattern elements: _* @els) => (tuple_pattern element: {els})), rule!( - (tuplePatternElement label: _? @label pattern: @p) + (tuplePatternElement label: _? @@label pattern: @p) => - (pattern_element key: {label.map(|l| tree!((identifier #{l})))} pattern: {p}) + (pattern_element key: (identifier #{label})? pattern: {p}) ), // A type-casting pattern (`case is T`). Not yet supported, so it is // mapped to `unsupported_node` — an explicit reminder that this needs @@ -626,26 +626,25 @@ fn translation_rules() -> Vec> { // The pattern-only shapes (`patternExpr`, `discardAssignmentExpr`) are // matched first; they never occur as ordinary call arguments. rule!( - (labeledExpr label: _? @lbl expression: (patternExpr pattern: @p)) + (labeledExpr label: _? @@lbl expression: (patternExpr pattern: @p)) => - (pattern_element key: {lbl.map(|l| tree!((identifier #{l})))} pattern: {p}) + (pattern_element key: (identifier #{lbl})? pattern: {p}) ), rule!( - (labeledExpr label: _? @lbl expression: (discardAssignmentExpr) @@wildcard) + (labeledExpr label: _? @@lbl expression: (discardAssignmentExpr) @@wildcard) => - (pattern_element key: {lbl.map(|l| tree!((identifier #{l})))} pattern: (ignore_pattern #{wildcard})) + (pattern_element key: (identifier #{lbl})? pattern: (ignore_pattern #{wildcard})) ), rule!( - (labeledExpr label: _? @lbl expression: @val) + (labeledExpr label: _? @@lbl expression: @val) => argument { - let key = lbl.map(|l| tree!((identifier #{l}))); if ctx.in_pattern { tree!((pattern_element - key: {key} + key: (identifier #{lbl})? pattern: (expr_equality_pattern expr: {val}))) } else { - tree!((argument name: {key} value: {val})) + tree!((argument name: (identifier #{lbl})? value: {val})) } } ), @@ -667,8 +666,8 @@ fn translation_rules() -> Vec> { // value; `break` / `continue` an optional target label; `throw` its // thrown expression. rule!((returnStmt expression: _? @val) => (return_expr value: {val})), - rule!((breakStmt label: _? @@lbl) => (break_expr label: {lbl.map(|l| tree!((identifier #{l})))})), - rule!((continueStmt label: _? @@lbl) => (continue_expr label: {lbl.map(|l| tree!((identifier #{l})))})), + rule!((breakStmt label: _? @@lbl) => (break_expr label: (identifier #{lbl})?)), + rule!((continueStmt label: _? @@lbl) => (continue_expr label: (identifier #{lbl})?)), rule!((throwStmt expression: @val) => (throw_expr value: {val})), // ---- Closures ---- // A closure (`{ (x: Int) -> Int in … }`) becomes a `function_expr`. The @@ -704,7 +703,7 @@ fn translation_rules() -> Vec> { initializer: (initializerClause value: @val)?) => (variable_declaration - modifier: {spec.map(|s| tree!((modifier #{s})))} + modifier: (modifier #{spec})? pattern: (name_pattern identifier: (identifier #{name})) value: {val}) ), @@ -948,7 +947,7 @@ fn translation_rules() -> Vec> { None => tree!((bulk_importing_pattern)), }; tree!((import_declaration - modifier: {kind.map(|k| tree!((modifier #{k})))} + modifier: (modifier #{kind})? modifier: {attrs} modifier: {mods} pattern: {pattern} @@ -1037,11 +1036,10 @@ fn translation_rules() -> Vec> { (tupleTypeElement firstName: _? @@name type: @ty) => tuple_type_element { - let name = name.map(|n| tree!((identifier #{n}))); if ctx.in_function_type { - tree!((parameter external_name: {name} type: {ty})) + tree!((parameter external_name: (identifier #{name})? type: {ty})) } else { - tree!((tuple_type_element name: {name} type: {ty})) + tree!((tuple_type_element name: (identifier #{name})? type: {ty})) } } ),