Update codegen/grammar

Cargo.lock

@@ -116,9 +116,11 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
 name = "ast-generator"
 version = "0.1.0"
 dependencies = [
- "itertools 0.10.5",
+ "either",
+ "itertools 0.12.1",
  "proc-macro2",
  "quote",
+ "ra_ap_stdx",
  "ungrammar",
 ]
 
@@ -902,15 +904,6 @@ version = "1.70.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf"
 
-[[package]]
-name = "itertools"
-version = "0.10.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473"
-dependencies = [
- "either",
-]
-
 [[package]]
 name = "itertools"
 version = "0.12.1"

rust/ast-generator/Cargo.toml

@@ -6,6 +6,8 @@ edition = "2021"
 # When updating these dependencies, run `rust/update_cargo_deps.sh`
 [dependencies]
 ungrammar = "1.16.1"
-proc-macro2 = "1.0.33"
-quote = "1.0.12"
-itertools = "0.10.1"
+proc-macro2 = "1.0.47"
+quote = "1.0.20"
+either = "1.9.0"
+ra_ap_stdx = "0.0.248"
+itertools = "0.12.0"

rust/ast-generator/src/codegen/grammar.rs

@@ -11,9 +11,11 @@ use std::{
     fs,
 };
 
+use either::Either;
 use itertools::Itertools;
 use proc_macro2::{Punct, Spacing};
 use quote::{format_ident, quote};
+use ra_ap_stdx::panic_context;
 use ungrammar::{Grammar, Rule};
 
 use crate::{
@@ -495,6 +497,7 @@ fn generate_syntax_kinds(grammar: KindsSrc) -> String {
         .map(|name| format_ident!("{}", name))
         .collect::<Vec<_>>();
 
+    // FIXME: This generates enum kinds?
     let nodes = grammar
         .nodes
         .iter()
@@ -757,6 +760,7 @@ pub(crate) fn lower(grammar: &Grammar) -> AstSrc {
     for &node in &nodes {
         let name = grammar[node].name.clone();
         let rule = &grammar[node].rule;
+        let _g = panic_context::enter(name.clone());
         match lower_enum(grammar, rule) {
             Some(variants) => {
                 let enum_src = AstEnumSrc {
@@ -904,11 +908,16 @@ fn lower_separated_list(
         Rule::Seq(it) => it,
         _ => return false,
     };
-    let (node, repeat, trailing_sep) = match rule.as_slice() {
+
+    let (nt, repeat, trailing_sep) = match rule.as_slice() {
         [Rule::Node(node), Rule::Rep(repeat), Rule::Opt(trailing_sep)] => {
-            (node, repeat, Some(trailing_sep))
+            (Either::Left(node), repeat, Some(trailing_sep))
         }
-        [Rule::Node(node), Rule::Rep(repeat)] => (node, repeat, None),
+        [Rule::Node(node), Rule::Rep(repeat)] => (Either::Left(node), repeat, None),
+        [Rule::Token(token), Rule::Rep(repeat), Rule::Opt(trailing_sep)] => {
+            (Either::Right(token), repeat, Some(trailing_sep))
+        }
+        [Rule::Token(token), Rule::Rep(repeat)] => (Either::Right(token), repeat, None),
         _ => return false,
     };
     let repeat = match &**repeat {
@@ -917,11 +926,22 @@ fn lower_separated_list(
     };
     if !matches!(
         repeat.as_slice(),
-        [comma, Rule::Node(n)]
-            if trailing_sep.map_or(true, |it| comma == &**it) && n == node
+        [comma, nt_]
+            if trailing_sep.map_or(true, |it| comma == &**it) && match (nt, nt_) {
+                (Either::Left(node), Rule::Node(nt_)) => node == nt_,
+                (Either::Right(token), Rule::Token(nt_)) => token == nt_,
+                _ => false,
+            }
     ) {
         return false;
     }
+    match nt {
+        Either::Right(token) => {
+            let name = clean_token_name(&grammar[*token].name);
+            let field = Field::Token(name);
+            acc.push(field);
+        }
+        Either::Left(node) => {
             let ty = grammar[*node].name.clone();
             let name = label
                 .cloned()
@@ -932,6 +952,8 @@ fn lower_separated_list(
                 cardinality: Cardinality::Many,
             };
             acc.push(field);
+        }
+    }
     true
 }
 

rust/ast-generator/src/codegen/grammar/ast_src.rs

@@ -118,9 +118,34 @@ const CONTEXTUAL_KEYWORDS: &[&str] = &[
     "dyn",
     "auto",
     "yeet",
+    "safe",
 ];
 // keywords we use for special macro expansions
-const CONTEXTUAL_BUILTIN_KEYWORDS: &[&str] = &["builtin", "offset_of", "format_args", "asm"];
+const CONTEXTUAL_BUILTIN_KEYWORDS: &[&str] = &[
+    "asm",
+    "att_syntax",
+    "builtin",
+    "clobber_abi",
+    "format_args",
+    // "in",
+    "inlateout",
+    "inout",
+    "label",
+    "lateout",
+    "may_unwind",
+    "nomem",
+    "noreturn",
+    "nostack",
+    "offset_of",
+    "options",
+    "out",
+    "preserves_flags",
+    "pure",
+    // "raw",
+    "readonly",
+    "sym",
+];
+
 // keywords that are keywords depending on the edition
 const EDITION_DEPENDENT_KEYWORDS: &[(&str, Edition)] = &[
     ("try", Edition::Edition2018),