From 8e7eedc172b38c8e65c8428a246b3fe9d5395c44 Mon Sep 17 00:00:00 2001 From: Arthur Baars Date: Mon, 16 Dec 2024 13:51:34 +0100 Subject: [PATCH] Update codegen/grammar --- Cargo.lock | 13 ++--- rust/ast-generator/Cargo.toml | 8 +-- rust/ast-generator/src/codegen/grammar.rs | 52 +++++++++++++------ .../src/codegen/grammar/ast_src.rs | 27 +++++++++- 4 files changed, 71 insertions(+), 29 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6ddab4533c7..e44c064e712 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -116,9 +116,11 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" name = "ast-generator" version = "0.1.0" dependencies = [ - "itertools 0.10.5", + "either", + "itertools 0.12.1", "proc-macro2", "quote", + "ra_ap_stdx", "ungrammar", ] @@ -902,15 +904,6 @@ version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" -[[package]] -name = "itertools" -version = "0.10.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" -dependencies = [ - "either", -] - [[package]] name = "itertools" version = "0.12.1" diff --git a/rust/ast-generator/Cargo.toml b/rust/ast-generator/Cargo.toml index 5eaff201b20..4060e501f92 100644 --- a/rust/ast-generator/Cargo.toml +++ b/rust/ast-generator/Cargo.toml @@ -6,6 +6,8 @@ edition = "2021" # When updating these dependencies, run `rust/update_cargo_deps.sh` [dependencies] ungrammar = "1.16.1" -proc-macro2 = "1.0.33" -quote = "1.0.12" -itertools = "0.10.1" +proc-macro2 = "1.0.47" +quote = "1.0.20" +either = "1.9.0" +ra_ap_stdx = "0.0.248" +itertools = "0.12.0" diff --git a/rust/ast-generator/src/codegen/grammar.rs b/rust/ast-generator/src/codegen/grammar.rs index 69875866ba5..bbafc435a5d 100644 --- a/rust/ast-generator/src/codegen/grammar.rs +++ b/rust/ast-generator/src/codegen/grammar.rs @@ -11,9 +11,11 @@ use std::{ fs, }; +use either::Either; use itertools::Itertools; use proc_macro2::{Punct, Spacing}; use quote::{format_ident, quote}; +use ra_ap_stdx::panic_context; use ungrammar::{Grammar, Rule}; use crate::{ @@ -495,6 +497,7 @@ fn generate_syntax_kinds(grammar: KindsSrc) -> String { .map(|name| format_ident!("{}", name)) .collect::>(); + // FIXME: This generates enum kinds? let nodes = grammar .nodes .iter() @@ -757,6 +760,7 @@ pub(crate) fn lower(grammar: &Grammar) -> AstSrc { for &node in &nodes { let name = grammar[node].name.clone(); let rule = &grammar[node].rule; + let _g = panic_context::enter(name.clone()); match lower_enum(grammar, rule) { Some(variants) => { let enum_src = AstEnumSrc { @@ -904,11 +908,16 @@ fn lower_separated_list( Rule::Seq(it) => it, _ => return false, }; - let (node, repeat, trailing_sep) = match rule.as_slice() { + + let (nt, repeat, trailing_sep) = match rule.as_slice() { [Rule::Node(node), Rule::Rep(repeat), Rule::Opt(trailing_sep)] => { - (node, repeat, Some(trailing_sep)) + (Either::Left(node), repeat, Some(trailing_sep)) } - [Rule::Node(node), Rule::Rep(repeat)] => (node, repeat, None), + [Rule::Node(node), Rule::Rep(repeat)] => (Either::Left(node), repeat, None), + [Rule::Token(token), Rule::Rep(repeat), Rule::Opt(trailing_sep)] => { + (Either::Right(token), repeat, Some(trailing_sep)) + } + [Rule::Token(token), Rule::Rep(repeat)] => (Either::Right(token), repeat, None), _ => return false, }; let repeat = match &**repeat { @@ -917,21 +926,34 @@ fn lower_separated_list( }; if !matches!( repeat.as_slice(), - [comma, Rule::Node(n)] - if trailing_sep.map_or(true, |it| comma == &**it) && n == node + [comma, nt_] + if trailing_sep.map_or(true, |it| comma == &**it) && match (nt, nt_) { + (Either::Left(node), Rule::Node(nt_)) => node == nt_, + (Either::Right(token), Rule::Token(nt_)) => token == nt_, + _ => false, + } ) { return false; } - let ty = grammar[*node].name.clone(); - let name = label - .cloned() - .unwrap_or_else(|| pluralize(&to_lower_snake_case(&ty))); - let field = Field::Node { - name, - ty, - cardinality: Cardinality::Many, - }; - acc.push(field); + match nt { + Either::Right(token) => { + let name = clean_token_name(&grammar[*token].name); + let field = Field::Token(name); + acc.push(field); + } + Either::Left(node) => { + let ty = grammar[*node].name.clone(); + let name = label + .cloned() + .unwrap_or_else(|| pluralize(&to_lower_snake_case(&ty))); + let field = Field::Node { + name, + ty, + cardinality: Cardinality::Many, + }; + acc.push(field); + } + } true } diff --git a/rust/ast-generator/src/codegen/grammar/ast_src.rs b/rust/ast-generator/src/codegen/grammar/ast_src.rs index 606a51b499c..88578ce7ebb 100644 --- a/rust/ast-generator/src/codegen/grammar/ast_src.rs +++ b/rust/ast-generator/src/codegen/grammar/ast_src.rs @@ -118,9 +118,34 @@ const CONTEXTUAL_KEYWORDS: &[&str] = &[ "dyn", "auto", "yeet", + "safe", ]; // keywords we use for special macro expansions -const CONTEXTUAL_BUILTIN_KEYWORDS: &[&str] = &["builtin", "offset_of", "format_args", "asm"]; +const CONTEXTUAL_BUILTIN_KEYWORDS: &[&str] = &[ + "asm", + "att_syntax", + "builtin", + "clobber_abi", + "format_args", + // "in", + "inlateout", + "inout", + "label", + "lateout", + "may_unwind", + "nomem", + "noreturn", + "nostack", + "offset_of", + "options", + "out", + "preserves_flags", + "pure", + // "raw", + "readonly", + "sym", +]; + // keywords that are keywords depending on the edition const EDITION_DEPENDENT_KEYWORDS: &[(&str, Edition)] = &[ ("try", Edition::Edition2018),