hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2026-06-30 09:05:28 +02:00
Code Issues Packages Projects Releases Wiki Activity

yeast: Replace {..expr} splice syntax with trait-dispatched {expr}

Browse Source 
In the initial implementation of yeast, the splice syntax was needed do
distinguish between splicing multiple nodes or just a single node.
However, this was always an ugly "wart" in the syntax, since the user
shouldn't have to worry about these things.

To fix this, we add an `IntoFieldIds` trait that dispatches on the
value's type: `Id` pushes a single id, and a blanket impl for
`IntoIterator<Item: Into<Id>>` handles `Vec<Id>`, `Option<Id>`, and
arbitrary iterator chains.

With this, we no longer need to use the special splice syntax, and hence
we can get rid of it.
This commit is contained in:
Taus
2026-06-26 12:45:35 +00:00
parent 9a5cc3c5e3
commit cc3c232631
6 changed files with 246 additions and 225 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
shared/yeast-macros/src/lib.rs
View File

@@ -41,15 +41,18 @@ pub fn query(input: TokenStream) -> TokenStream {
/// (kind "literal") - leaf with static content
/// (kind #{expr}) - leaf with computed content (expr.to_string())
/// (kind $fresh) - leaf with auto-generated unique name
/// {expr} - embed a Rust expression returning Id
/// {..expr} - splice an iterable of Id (in child/field position)
/// field: {..expr} - splice into a named field
/// {expr} - embed a Rust expression, dispatched via
/// the `IntoFieldIds` trait: `Id` pushes a
/// single id; iterables (`Vec<Id>`,
/// `Option<Id>`, iterator chains) splice
/// their elements
/// field: {expr} - extend a named field with `{expr}`'s ids
/// {expr}.map(p -> tpl) - apply tpl to each element; splice result
/// {expr}.reduce_left(f -> init, acc, e -> fold)
/// - fold with per-element init; splice 0 or 1 result
/// ```
///
/// Chain syntax after `{expr}` or `{..expr}`:
/// Chain syntax after `{expr}`:
/// - `.map(param -> template)` — one output node per input element.
/// - `.reduce_left(first -> init, acc, elem -> fold)` — fold left; the first
/// element is converted by `init`, subsequent elements are folded by `fold`
@@ -100,7 +103,7 @@ pub fn trees(input: TokenStream) -> TokenStream {
/// rule!(
/// (query_pattern field: (_) @name (kind)* @repeated (_)? @optional)
/// =>
/// (output_template field: {name} {..repeated})
/// (output_template field: {name} {repeated})
/// )
///
/// // Shorthand: captures become fields on the output node

100
shared/yeast-macros/src/parse.rs
View File

@@ -429,45 +429,41 @@ fn parse_direct_node_inner(tokens: &mut Tokens, ctx: &Ident) -> Result<TokenStre
);
field_counter += 1;
// Check for field: {..expr}.chain or field: {expr}.chain — splice a Vec<Id> into the field
// Check for field: {expr}.chain (chain pipeline) or plain field: {expr}
// (trait-dispatched: handles single ids and iterables uniformly).
if peek_is_group(tokens, Delimiter::Brace) {
let group_clone = tokens.clone().next().unwrap();
if let TokenTree::Group(g) = &group_clone {
let mut inner_check = g.stream().into_iter();
let is_splice = matches!(inner_check.next(), Some(TokenTree::Punct(p)) if p.as_char() == '.')
&& matches!(inner_check.next(), Some(TokenTree::Punct(p)) if p.as_char() == '.');
// Determine if a chain (.map(..)) follows the `{}` group.
let mut after = tokens.clone();
after.next(); // skip the brace group
let has_chain =
matches!(after.peek(), Some(TokenTree::Punct(p)) if p.as_char() == '.');
// Determine if a chain (.map(..)) follows the `{}` group.
let mut after = tokens.clone();
after.next(); // skip the brace group
let has_chain = matches!(after.peek(), Some(TokenTree::Punct(p)) if p.as_char() == '.');
if is_splice || has_chain {
let group = expect_group(tokens, Delimiter::Brace)?;
let base: TokenStream = if is_splice {
let mut inner = group.stream().into_iter().peekable();
inner.next(); // consume first .
inner.next(); // consume second .
let expr: TokenStream = inner.collect();
quote! {
{ #expr }.into_iter().map(::std::convert::Into::<yeast::Id>::into)
}
} else {
let expr = group.stream();
quote! { { #expr }.into_iter() }
};
let chained = parse_chain_suffix(tokens, ctx, base)?;
stmts.push(quote! {
let #temp: Vec<yeast::Id> = #chained.collect();
});
// An empty splice means the field is absent — skip it
// entirely rather than emitting an empty named field.
field_args.push(quote! {
if !#temp.is_empty() { __fields.push((#field_str, #temp)); }
});
continue;
}
if has_chain {
let group = expect_group(tokens, Delimiter::Brace)?;
let expr = group.stream();
let base = quote! { { #expr }.into_iter() };
let chained = parse_chain_suffix(tokens, ctx, base)?;
stmts.push(quote! {
let #temp: Vec<yeast::Id> = #chained.collect();
});
// An empty pipeline means the field is absent — skip it
// entirely rather than emitting an empty named field.
field_args.push(quote! {
if !#temp.is_empty() { __fields.push((#field_str, #temp)); }
});
continue;
}
// Plain `{expr}` — trait-dispatched extend.
let group = expect_group(tokens, Delimiter::Brace)?;
let expr = group.stream();
stmts.push(quote! {
let mut #temp: Vec<yeast::Id> = Vec::new();
yeast::IntoFieldIds::extend_into({ #expr }, &mut #temp);
});
field_args.push(quote! {
if !#temp.is_empty() { __fields.push((#field_str, #temp)); }
});
continue;
}
let value = parse_direct_node(tokens, ctx)?;
@@ -495,8 +491,7 @@ fn parse_direct_node_inner(tokens: &mut Tokens, ctx: &Ident) -> Result<TokenStre
})
}
/// Parse a chain of `.method(args)` suffixes after a `{expr}` or `{..expr}`
/// placeholder in tree templates. Currently supports:
/// Parse a chain of `.method(args)` suffixes after a `{expr}` placeholder in tree templates. Currently supports:
///
/// ```text
/// .map(param -> template) -- iterator map: produces Vec<yeast::Id>
@@ -603,25 +598,15 @@ fn parse_direct_list(tokens: &mut Tokens, ctx: &Ident) -> Result<Vec<TokenStream
continue;
}
// {expr} or {..expr} (with optional .chain) — single node or splice
// {expr} (with optional `.chain` pipeline) — extend `__nodes` via
// `IntoFieldIds`, which handles single ids and iterables uniformly.
if peek_is_group(tokens, Delimiter::Brace) {
let group = expect_group(tokens, Delimiter::Brace)?;
let has_chain =
matches!(tokens.peek(), Some(TokenTree::Punct(p)) if p.as_char() == '.');
let mut inner = group.stream().into_iter().peekable();
let is_splice = peek_is_dotdot(&inner);
if is_splice || has_chain {
let base: TokenStream = if is_splice {
inner.next(); // consume first .
inner.next(); // consume second .
let expr: TokenStream = inner.collect();
quote! {
{ #expr }.into_iter().map(::std::convert::Into::<yeast::Id>::into)
}
} else {
let expr = group.stream();
quote! { { #expr }.into_iter() }
};
if has_chain {
let expr = group.stream();
let base = quote! { { #expr }.into_iter() };
let chained = parse_chain_suffix(tokens, ctx, base)?;
items.push(quote! {
__nodes.extend(#chained);
@@ -629,7 +614,7 @@ fn parse_direct_list(tokens: &mut Tokens, ctx: &Ident) -> Result<Vec<TokenStream
} else {
let expr = group.stream();
items.push(quote! {
__nodes.push(::std::convert::Into::<yeast::Id>::into({ #expr }));
yeast::IntoFieldIds::extend_into({ #expr }, &mut __nodes);
});
}
continue;
@@ -951,13 +936,6 @@ fn peek_is_hash(tokens: &mut Tokens) -> bool {
matches!(tokens.peek(), Some(TokenTree::Punct(p)) if p.as_char() == '#')
}
/// Check for `..` (two consecutive dot punctuation tokens).
fn peek_is_dotdot(tokens: &Tokens) -> bool {
let mut lookahead = tokens.clone();
matches!(lookahead.next(), Some(TokenTree::Punct(p)) if p.as_char() == '.')
&& matches!(lookahead.next(), Some(TokenTree::Punct(p)) if p.as_char() == '.')
}
fn peek_is_underscore(tokens: &mut Tokens) -> bool {
matches!(tokens.peek(), Some(TokenTree::Ident(id)) if *id == "_")
}

38
shared/yeast/doc/yeast.md
View File

@@ -214,7 +214,7 @@ yeast::tree!(ctx,
```rust
yeast::trees!(ctx,
(assignment left: {tmp} right: {right})
{..body}
{body}
)
```
@@ -256,12 +256,26 @@ occurrences of the same `$name` within one `BuildCtx` share the same value:
### Embedded Rust expressions
`{expr}` embeds a Rust expression that returns a single node `Id`:
`{expr}` embeds a Rust expression whose value is appended to the
enclosing field (or to the rule body's id list). Dispatch happens via
the [`IntoFieldIds`] trait, which is implemented for:
- `Id` — pushes the single id.
- Any `IntoIterator<Item: Into<Id>>` — extends with all yielded ids
(covers `Vec<Id>`, `Option<Id>`, iterator chains, etc.).
So the same `{expr}` syntax handles single ids, splices, and zero-or-many
options uniformly:
```rust
(assignment
left: {some_node_id} // insert a pre-built node
right: {rhs} // insert a captured value (inside rule!)
left: {some_node_id} // a single Id
right: {rhs} // a captured value (inside rule!)
)
yeast::trees!(ctx,
(assignment left: {tmp} right: {right})
{extra_nodes} // splices a Vec<Id>
)
```
@@ -277,21 +291,17 @@ expressions (with `let` bindings) work too:
})
```
`{..expr}` splices a `Vec<Id>` (or any iterable of `Id`); the contents
are likewise a Rust block, so the splice can be the result of arbitrary
computation:
Inside `rule!`, captures are Rust variables — `{name}` works for
single, optional, and repeated captures alike:
```rust
yeast::trees!(ctx,
(assignment left: {tmp} right: {right})
{..extra_nodes} // splice a Vec<Id>
rule!(
(assignment left: @lhs right: _* @parts)
=>
(assignment left: {lhs} right: (block stmt: {parts}))
)
```
Inside `rule!`, captures are Rust variables, so `{name}` inserts a
single capture (`Id`) and `{..name}` splices a repeated capture
(`Vec<Id>`).
### Raw captures (`@@name`)
The default `@name` capture marker is *auto-translated*: in OneShot

30
shared/yeast/src/lib.rs
View File

@@ -48,6 +48,36 @@ impl From<Id> for usize {
type FieldId = u16;
type KindId = u16;
/// Trait for values that can be appended to a field's id list inside a
/// `tree!`/`trees!`/`rule!` template (in `{expr}` placeholders).
///
/// `Id` pushes a single id; the blanket impl for
/// `IntoIterator<Item: Into<Id>>` handles `Vec<Id>`, `Option<Id>`,
/// arbitrary iterators yielding `Id`, etc.
///
/// This lets `{expr}` interpolate any of these shapes without a
/// dedicated splice syntax — the macro emits the same trait-dispatched
/// call regardless of the value's type.
pub trait IntoFieldIds {
fn extend_into(self, out: &mut Vec<Id>);
}
impl IntoFieldIds for Id {
fn extend_into(self, out: &mut Vec<Id>) {
out.push(self);
}
}
impl<I, T> IntoFieldIds for I
where
I: IntoIterator<Item = T>,
T: Into<Id>,
{
fn extend_into(self, out: &mut Vec<Id>) {
out.extend(self.into_iter().map(Into::into));
}
}
/// Like [`std::fmt::Display`], but the formatting routine is given access to
/// the [`Ast`] so that node references can resolve to their source text.
///

16
shared/yeast/tests/test.rs
View File

@@ -635,7 +635,7 @@ fn ruby_rules() -> Vec<Rule> {
left: (identifier $tmp)
right: {right}
)
{..left.iter().enumerate().map(|(i, &lhs)|
{left.iter().enumerate().map(|(i, &lhs)|
yeast::tree!(
(assignment
left: {lhs}
@@ -667,7 +667,7 @@ fn ruby_rules() -> Vec<Rule> {
left: {pat}
right: (identifier $tmp)
)
stmt: {..body}
stmt: {body}
)
)
)
@@ -903,7 +903,7 @@ fn one_shot_xeq1_rules() -> Vec<Rule> {
yeast::rule!(
(program (_)* @stmts)
=>
(program stmt: {..stmts})
(program stmt: {stmts})
),
yeast::rule!(
(assignment left: (_) @left right: (_) @right)
@@ -979,7 +979,7 @@ fn test_one_shot_recurses_into_returned_capture() {
yeast::rule!(
(program (_)* @stmts)
=>
(program stmt: {..stmts})
(program stmt: {stmts})
),
// Returns the captured `left` verbatim, discarding `right`.
yeast::rule!(
@@ -1021,7 +1021,7 @@ fn test_one_shot_does_not_recurse_into_wrapper_output() {
yeast::rule!(
(program (_)* @stmts)
=>
(program stmt: {..stmts})
(program stmt: {stmts})
),
// Wraps `left` in nested `first_node`/`second_node` output kinds.
// Neither wrapper kind has a matching rule, so a buggy implementation
@@ -1072,7 +1072,7 @@ fn test_raw_capture_marker() {
yeast::rule!(
(program (_)* @stmts)
=>
(program stmt: {..stmts})
(program stmt: {stmts})
),
// `@@raw_lhs` is untranslated: the body reads its source text
// ("x") and embeds it directly as the identifier content. `@rhs`
@@ -1130,7 +1130,7 @@ fn test_raw_capture_marker_explicit_translate() {
yeast::rule!(
(program (_)* @stmts)
=>
(program stmt: {..stmts})
(program stmt: {stmts})
),
yeast::rule!(
(assignment left: (_) @@raw_lhs right: (_) @rhs)
@@ -1138,7 +1138,7 @@ fn test_raw_capture_marker_explicit_translate() {
{
let translated_lhs = ctx.translate(raw_lhs)?;
tree!((call
method: {..translated_lhs}
method: {translated_lhs}
receiver: {rhs}))
}
),