hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2025-12-16 16:53:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #18429 from github/redsun82/rust-ast-generator-mustache

Browse Source 
Rust: make ast-generator use mustache templates
This commit is contained in:
Paolo Tranquilli
2025-01-09 08:37:07 +01:00
committed by GitHub
parent 554ea29547 53b0a3464d
commit ca05697365
21 changed files with 1388 additions and 1122 deletions
Download Patch File Download Diff File Expand all files Collapse all files

52
Cargo.lock generated
View File

@@ -116,11 +116,14 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
name = "ast-generator"
version = "0.1.0"
dependencies = [
"anyhow",
"either",
"itertools 0.14.0",
"mustache",
"proc-macro2",
"quote",
"ra_ap_stdx",
"serde",
"ungrammar",
]
@@ -168,9 +171,9 @@ dependencies = [
[[package]]
name = "bstr"
version = "1.11.1"
version = "1.11.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "786a307d683a5bf92e6fd5fd69a7eb613751668d1d8d67d802846dfe367c62c8"
checksum = "531a9155a481e2ee699d4f98f43c0ca4ff8ee1bfd55c31e9e98fb29d2b176fe0"
dependencies = [
"memchr",
"serde",
@@ -184,9 +187,9 @@ checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c"
[[package]]
name = "bytemuck"
version = "1.20.0"
version = "1.21.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b37c88a63ffd85d15b406896cc343916d7cf57838a847b3a6f2ca5d39a5695a"
checksum = "ef657dfab802224e671f5818e9a4935f9b1957ed18e58292690cc39e7a4092a3"
[[package]]
name = "byteorder"
@@ -412,7 +415,7 @@ dependencies = [
"figment",
"glob",
"itertools 0.14.0",
"log",
"log 0.4.22",
"num-traits",
"ra_ap_base_db",
"ra_ap_cfg",
@@ -585,7 +588,7 @@ version = "0.14.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d248bdd43ce613d87415282f69b9bb99d947d290b10962dd6c56233312c2ad5"
dependencies = [
"log",
"log 0.4.22",
]
[[package]]
@@ -755,7 +758,7 @@ checksum = "15f1ce686646e7f1e19bf7d5533fe443a45dbfb990e00629110797578b42fb19"
dependencies = [
"aho-corasick",
"bstr",
"log",
"log 0.4.22",
"regex-automata 0.4.9",
"regex-syntax 0.8.5",
]
@@ -1020,6 +1023,15 @@ dependencies = [
"scopeguard",
]
[[package]]
name = "log"
version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e19e8d5c34a3e0e2223db8e060f9e8264aeeb5c5fc64a4ee9965c062211c024b"
dependencies = [
"log 0.4.22",
]
[[package]]
name = "log"
version = "0.4.22"
@@ -1072,7 +1084,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c"
dependencies = [
"libc",
"log",
"log 0.4.22",
"wasi",
"windows-sys 0.48.0",
]
@@ -1086,6 +1098,16 @@ dependencies = [
"windows-sys 0.48.0",
]
[[package]]
name = "mustache"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "51956ef1c5d20a1384524d91e616fb44dfc7d8f249bf696d49c97dd3289ecab5"
dependencies = [
"log 0.3.9",
"serde",
]
[[package]]
name = "nohash-hasher"
version = "0.2.0"
@@ -1105,7 +1127,7 @@ dependencies = [
"inotify",
"kqueue",
"libc",
"log",
"log 0.4.22",
"mio",
"walkdir",
"windows-sys 0.48.0",
@@ -2160,7 +2182,7 @@ checksum = "61c910772f992ab17d32d6760e167d2353f4130ed50e796752689556af07dc6b"
dependencies = [
"chrono",
"is-terminal",
"log",
"log 0.4.22",
"termcolor",
"thread_local",
]
@@ -2313,7 +2335,7 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3"
dependencies = [
"log",
"log 0.4.22",
"once_cell",
"tracing-core",
]
@@ -2338,9 +2360,9 @@ dependencies = [
[[package]]
name = "tree-sitter"
version = "0.24.5"
version = "0.24.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ac95b18f0f727aaaa012bd5179a1916706ee3ed071920fdbda738750b0c0bf5"
checksum = "5f2434c86ba59ed15af56039cc5bf1acf8ba76ce301e32ef08827388ef285ec5"
dependencies = [
"cc",
"regex",
@@ -2502,7 +2524,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5f89bb38646b4f81674e8f5c3fb81b562be1fd936d84320f3264486418519c79"
dependencies = [
"bumpalo",
"log",
"log 0.4.22",
"proc-macro2",
"quote",
"syn",
@@ -2560,7 +2582,7 @@ version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb"
dependencies = [
"windows-sys 0.48.0",
"windows-sys 0.59.0",
]
[[package]]

2
MODULE.bazel
View File

@@ -70,7 +70,7 @@ use_repo(py_deps, "vendor__anyhow-1.0.44", "vendor__cc-1.0.70", "vendor__clap-2.
# deps for ruby+rust
# keep in sync by running `misc/bazel/3rdparty/update_cargo_deps.sh`
tree_sitter_extractors_deps = use_extension("//misc/bazel/3rdparty:tree_sitter_extractors_extension.bzl", "r")
use_repo(tree_sitter_extractors_deps, "vendor__anyhow-1.0.95", "vendor__argfile-0.2.1", "vendor__chrono-0.4.39", "vendor__clap-4.5.24", "vendor__dunce-1.0.5", "vendor__either-1.13.0", "vendor__encoding-0.2.33", "vendor__figment-0.10.19", "vendor__flate2-1.0.35", "vendor__glob-0.3.2", "vendor__globset-0.4.15", "vendor__itertools-0.14.0", "vendor__lazy_static-1.5.0", "vendor__log-0.4.22", "vendor__num-traits-0.2.19", "vendor__num_cpus-1.16.0", "vendor__proc-macro2-1.0.92", "vendor__quote-1.0.38", "vendor__ra_ap_base_db-0.0.248", "vendor__ra_ap_cfg-0.0.248", "vendor__ra_ap_hir-0.0.248", "vendor__ra_ap_hir_def-0.0.248", "vendor__ra_ap_hir_expand-0.0.248", "vendor__ra_ap_ide_db-0.0.248", "vendor__ra_ap_intern-0.0.248", "vendor__ra_ap_load-cargo-0.0.248", "vendor__ra_ap_parser-0.0.248", "vendor__ra_ap_paths-0.0.248", "vendor__ra_ap_project_model-0.0.248", "vendor__ra_ap_span-0.0.248", "vendor__ra_ap_stdx-0.0.248", "vendor__ra_ap_syntax-0.0.248", "vendor__ra_ap_vfs-0.0.248", "vendor__rand-0.8.5", "vendor__rayon-1.10.0", "vendor__regex-1.11.1", "vendor__serde-1.0.217", "vendor__serde_json-1.0.135", "vendor__serde_with-3.12.0", "vendor__stderrlog-0.6.0", "vendor__syn-2.0.95", "vendor__tracing-0.1.41", "vendor__tracing-subscriber-0.3.19", "vendor__tree-sitter-0.24.5", "vendor__tree-sitter-embedded-template-0.23.2", "vendor__tree-sitter-json-0.24.8", "vendor__tree-sitter-ql-0.23.1", "vendor__tree-sitter-ruby-0.23.1", "vendor__triomphe-0.1.14", "vendor__ungrammar-1.16.1")
use_repo(tree_sitter_extractors_deps, "vendor__anyhow-1.0.95", "vendor__argfile-0.2.1", "vendor__chrono-0.4.39", "vendor__clap-4.5.24", "vendor__dunce-1.0.5", "vendor__either-1.13.0", "vendor__encoding-0.2.33", "vendor__figment-0.10.19", "vendor__flate2-1.0.35", "vendor__glob-0.3.2", "vendor__globset-0.4.15", "vendor__itertools-0.14.0", "vendor__lazy_static-1.5.0", "vendor__log-0.4.22", "vendor__mustache-0.9.0", "vendor__num-traits-0.2.19", "vendor__num_cpus-1.16.0", "vendor__proc-macro2-1.0.92", "vendor__quote-1.0.38", "vendor__ra_ap_base_db-0.0.248", "vendor__ra_ap_cfg-0.0.248", "vendor__ra_ap_hir-0.0.248", "vendor__ra_ap_hir_def-0.0.248", "vendor__ra_ap_hir_expand-0.0.248", "vendor__ra_ap_ide_db-0.0.248", "vendor__ra_ap_intern-0.0.248", "vendor__ra_ap_load-cargo-0.0.248", "vendor__ra_ap_parser-0.0.248", "vendor__ra_ap_paths-0.0.248", "vendor__ra_ap_project_model-0.0.248", "vendor__ra_ap_span-0.0.248", "vendor__ra_ap_stdx-0.0.248", "vendor__ra_ap_syntax-0.0.248", "vendor__ra_ap_vfs-0.0.248", "vendor__rand-0.8.5", "vendor__rayon-1.10.0", "vendor__regex-1.11.1", "vendor__serde-1.0.217", "vendor__serde_json-1.0.135", "vendor__serde_with-3.12.0", "vendor__stderrlog-0.6.0", "vendor__syn-2.0.95", "vendor__tracing-0.1.41", "vendor__tracing-subscriber-0.3.19", "vendor__tree-sitter-0.24.6", "vendor__tree-sitter-embedded-template-0.23.2", "vendor__tree-sitter-json-0.24.8", "vendor__tree-sitter-ql-0.23.1", "vendor__tree-sitter-ruby-0.23.1", "vendor__triomphe-0.1.14", "vendor__ungrammar-1.16.1")
http_archive = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")

2
misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.atomic-0.6.0.bazel generated vendored
View File

@@ -83,6 +83,6 @@ rust_library(
}),
version = "0.6.0",
deps = [
"@vendor__bytemuck-1.20.0//:bytemuck",
"@vendor__bytemuck-1.21.0//:bytemuck",
],
)

8
misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.bazel generated vendored
View File

@@ -115,6 +115,12 @@ alias(
tags = ["manual"],
)
alias(
name = "mustache",
actual = "@vendor__mustache-0.9.0//:mustache",
tags = ["manual"],
)
alias(
name = "num-traits",
actual = "@vendor__num-traits-0.2.19//:num_traits",
@@ -291,7 +297,7 @@ alias(
alias(
name = "tree-sitter",
actual = "@vendor__tree-sitter-0.24.5//:tree_sitter",
actual = "@vendor__tree-sitter-0.24.6//:tree_sitter",
tags = ["manual"],
)

2
misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.bstr-1.11.1.bazel → misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.bstr-1.11.3.bazel generated vendored
View File

@@ -81,7 +81,7 @@ rust_library(
"@rules_rust//rust/platform:x86_64-unknown-none": [],
"//conditions:default": ["@platforms//:incompatible"],
}),
version = "1.11.1",
version = "1.11.3",
deps = [
"@vendor__memchr-2.7.4//:memchr",
],

2
misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.bytemuck-1.20.0.bazel → misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.bytemuck-1.21.0.bazel generated vendored
View File

@@ -77,5 +77,5 @@ rust_library(
"@rules_rust//rust/platform:x86_64-unknown-none": [],
"//conditions:default": ["@platforms//:incompatible"],
}),
version = "1.20.0",
version = "1.21.0",
)

2
misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.globset-0.4.15.bazel generated vendored
View File

@@ -84,7 +84,7 @@ rust_library(
version = "0.4.15",
deps = [
"@vendor__aho-corasick-1.1.3//:aho_corasick",
"@vendor__bstr-1.11.1//:bstr",
"@vendor__bstr-1.11.3//:bstr",
"@vendor__log-0.4.22//:log",
"@vendor__regex-automata-0.4.9//:regex_automata",
"@vendor__regex-syntax-0.8.5//:regex_syntax",

88
misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.log-0.3.9.bazel generated vendored Normal file
View File

@@ -0,0 +1,88 @@
###############################################################################
# @generated
# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To
# regenerate this file, run the following:
#
# bazel run @@//misc/bazel/3rdparty:vendor_tree_sitter_extractors
###############################################################################
load("@rules_rust//rust:defs.bzl", "rust_library")
package(default_visibility = ["//visibility:public"])
rust_library(
name = "log",
srcs = glob(
include = ["**/*.rs"],
allow_empty = True,
),
compile_data = glob(
include = ["**"],
allow_empty = True,
exclude = [
"**/* *",
".tmp_git_root/**/*",
"BUILD",
"BUILD.bazel",
"WORKSPACE",
"WORKSPACE.bazel",
],
),
crate_features = [
"default",
"use_std",
],
crate_root = "src/lib.rs",
edition = "2015",
rustc_flags = [
"--cap-lints=allow",
],
tags = [
"cargo-bazel",
"crate-name=log",
"manual",
"noclippy",
"norustfmt",
],
target_compatible_with = select({
"@rules_rust//rust/platform:aarch64-apple-darwin": [],
"@rules_rust//rust/platform:aarch64-apple-ios": [],
"@rules_rust//rust/platform:aarch64-apple-ios-sim": [],
"@rules_rust//rust/platform:aarch64-fuchsia": [],
"@rules_rust//rust/platform:aarch64-linux-android": [],
"@rules_rust//rust/platform:aarch64-pc-windows-msvc": [],
"@rules_rust//rust/platform:aarch64-unknown-linux-gnu": [],
"@rules_rust//rust/platform:aarch64-unknown-nixos-gnu": [],
"@rules_rust//rust/platform:aarch64-unknown-nto-qnx710": [],
"@rules_rust//rust/platform:arm-unknown-linux-gnueabi": [],
"@rules_rust//rust/platform:armv7-linux-androideabi": [],
"@rules_rust//rust/platform:armv7-unknown-linux-gnueabi": [],
"@rules_rust//rust/platform:i686-apple-darwin": [],
"@rules_rust//rust/platform:i686-linux-android": [],
"@rules_rust//rust/platform:i686-pc-windows-msvc": [],
"@rules_rust//rust/platform:i686-unknown-freebsd": [],
"@rules_rust//rust/platform:i686-unknown-linux-gnu": [],
"@rules_rust//rust/platform:powerpc-unknown-linux-gnu": [],
"@rules_rust//rust/platform:riscv32imc-unknown-none-elf": [],
"@rules_rust//rust/platform:riscv64gc-unknown-none-elf": [],
"@rules_rust//rust/platform:s390x-unknown-linux-gnu": [],
"@rules_rust//rust/platform:thumbv7em-none-eabi": [],
"@rules_rust//rust/platform:thumbv8m.main-none-eabi": [],
"@rules_rust//rust/platform:wasm32-unknown-unknown": [],
"@rules_rust//rust/platform:wasm32-wasi": [],
"@rules_rust//rust/platform:x86_64-apple-darwin": [],
"@rules_rust//rust/platform:x86_64-apple-ios": [],
"@rules_rust//rust/platform:x86_64-fuchsia": [],
"@rules_rust//rust/platform:x86_64-linux-android": [],
"@rules_rust//rust/platform:x86_64-pc-windows-msvc": [],
"@rules_rust//rust/platform:x86_64-unknown-freebsd": [],
"@rules_rust//rust/platform:x86_64-unknown-linux-gnu": [],
"@rules_rust//rust/platform:x86_64-unknown-nixos-gnu": [],
"@rules_rust//rust/platform:x86_64-unknown-none": [],
"//conditions:default": ["@platforms//:incompatible"],
}),
version = "0.3.9",
deps = [
"@vendor__log-0.4.22//:log",
],
)

85
misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.mustache-0.9.0.bazel generated vendored Normal file
View File

@@ -0,0 +1,85 @@
###############################################################################
# @generated
# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To
# regenerate this file, run the following:
#
# bazel run @@//misc/bazel/3rdparty:vendor_tree_sitter_extractors
###############################################################################
load("@rules_rust//rust:defs.bzl", "rust_library")
package(default_visibility = ["//visibility:public"])
rust_library(
name = "mustache",
srcs = glob(
include = ["**/*.rs"],
allow_empty = True,
),
compile_data = glob(
include = ["**"],
allow_empty = True,
exclude = [
"**/* *",
".tmp_git_root/**/*",
"BUILD",
"BUILD.bazel",
"WORKSPACE",
"WORKSPACE.bazel",
],
),
crate_root = "src/lib.rs",
edition = "2015",
rustc_flags = [
"--cap-lints=allow",
],
tags = [
"cargo-bazel",
"crate-name=mustache",
"manual",
"noclippy",
"norustfmt",
],
target_compatible_with = select({
"@rules_rust//rust/platform:aarch64-apple-darwin": [],
"@rules_rust//rust/platform:aarch64-apple-ios": [],
"@rules_rust//rust/platform:aarch64-apple-ios-sim": [],
"@rules_rust//rust/platform:aarch64-fuchsia": [],
"@rules_rust//rust/platform:aarch64-linux-android": [],
"@rules_rust//rust/platform:aarch64-pc-windows-msvc": [],
"@rules_rust//rust/platform:aarch64-unknown-linux-gnu": [],
"@rules_rust//rust/platform:aarch64-unknown-nixos-gnu": [],
"@rules_rust//rust/platform:aarch64-unknown-nto-qnx710": [],
"@rules_rust//rust/platform:arm-unknown-linux-gnueabi": [],
"@rules_rust//rust/platform:armv7-linux-androideabi": [],
"@rules_rust//rust/platform:armv7-unknown-linux-gnueabi": [],
"@rules_rust//rust/platform:i686-apple-darwin": [],
"@rules_rust//rust/platform:i686-linux-android": [],
"@rules_rust//rust/platform:i686-pc-windows-msvc": [],
"@rules_rust//rust/platform:i686-unknown-freebsd": [],
"@rules_rust//rust/platform:i686-unknown-linux-gnu": [],
"@rules_rust//rust/platform:powerpc-unknown-linux-gnu": [],
"@rules_rust//rust/platform:riscv32imc-unknown-none-elf": [],
"@rules_rust//rust/platform:riscv64gc-unknown-none-elf": [],
"@rules_rust//rust/platform:s390x-unknown-linux-gnu": [],
"@rules_rust//rust/platform:thumbv7em-none-eabi": [],
"@rules_rust//rust/platform:thumbv8m.main-none-eabi": [],
"@rules_rust//rust/platform:wasm32-unknown-unknown": [],
"@rules_rust//rust/platform:wasm32-wasi": [],
"@rules_rust//rust/platform:x86_64-apple-darwin": [],
"@rules_rust//rust/platform:x86_64-apple-ios": [],
"@rules_rust//rust/platform:x86_64-fuchsia": [],
"@rules_rust//rust/platform:x86_64-linux-android": [],
"@rules_rust//rust/platform:x86_64-pc-windows-msvc": [],
"@rules_rust//rust/platform:x86_64-unknown-freebsd": [],
"@rules_rust//rust/platform:x86_64-unknown-linux-gnu": [],
"@rules_rust//rust/platform:x86_64-unknown-nixos-gnu": [],
"@rules_rust//rust/platform:x86_64-unknown-none": [],
"//conditions:default": ["@platforms//:incompatible"],
}),
version = "0.9.0",
deps = [
"@vendor__log-0.3.9//:log",
"@vendor__serde-1.0.217//:serde",
],
)

6
misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.tree-sitter-0.24.5.bazel → misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.tree-sitter-0.24.6.bazel generated vendored
View File

@@ -82,12 +82,12 @@ rust_library(
"@rules_rust//rust/platform:x86_64-unknown-none": [],
"//conditions:default": ["@platforms//:incompatible"],
}),
version = "0.24.5",
version = "0.24.6",
deps = [
"@vendor__regex-1.11.1//:regex",
"@vendor__regex-syntax-0.8.5//:regex_syntax",
"@vendor__streaming-iterator-0.1.9//:streaming_iterator",
"@vendor__tree-sitter-0.24.5//:build_script_build",
"@vendor__tree-sitter-0.24.6//:build_script_build",
"@vendor__tree-sitter-language-0.1.3//:tree_sitter_language",
],
)
@@ -142,7 +142,7 @@ cargo_build_script(
"noclippy",
"norustfmt",
],
version = "0.24.5",
version = "0.24.6",
visibility = ["//visibility:private"],
deps = [
"@vendor__cc-1.2.7//:cc",

6
misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.winapi-util-0.1.9.bazel generated vendored
View File

@@ -80,13 +80,13 @@ rust_library(
version = "0.1.9",
deps = select({
"@rules_rust//rust/platform:aarch64-pc-windows-msvc": [
"@vendor__windows-sys-0.48.0//:windows_sys", # cfg(windows)
"@vendor__windows-sys-0.59.0//:windows_sys", # cfg(windows)
],
"@rules_rust//rust/platform:i686-pc-windows-msvc": [
"@vendor__windows-sys-0.48.0//:windows_sys", # cfg(windows)
"@vendor__windows-sys-0.59.0//:windows_sys", # cfg(windows)
],
"@rules_rust//rust/platform:x86_64-pc-windows-msvc": [
"@vendor__windows-sys-0.48.0//:windows_sys", # cfg(windows)
"@vendor__windows-sys-0.59.0//:windows_sys", # cfg(windows)
],
"//conditions:default": [],
}),

2
misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.windows-sys-0.48.0.bazel generated vendored
View File

@@ -37,10 +37,8 @@ rust_library(
"Win32_Storage",
"Win32_Storage_FileSystem",
"Win32_System",
"Win32_System_Console",
"Win32_System_IO",
"Win32_System_Pipes",
"Win32_System_SystemInformation",
"Win32_System_Threading",
"Win32_System_WindowsProgramming",
"default",

1
misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.windows-sys-0.59.0.bazel generated vendored
View File

@@ -36,6 +36,7 @@ rust_library(
"Win32_System",
"Win32_System_Com",
"Win32_System_Console",
"Win32_System_SystemInformation",
"Win32_UI",
"Win32_UI_Shell",
"default",

60
misc/bazel/3rdparty/tree_sitter_extractors_deps/defs.bzl generated vendored
View File

@@ -303,17 +303,20 @@ _NORMAL_DEPENDENCIES = {
"regex": Label("@vendor__regex-1.11.1//:regex"),
"tracing": Label("@vendor__tracing-0.1.41//:tracing"),
"tracing-subscriber": Label("@vendor__tracing-subscriber-0.3.19//:tracing_subscriber"),
"tree-sitter": Label("@vendor__tree-sitter-0.24.5//:tree_sitter"),
"tree-sitter": Label("@vendor__tree-sitter-0.24.6//:tree_sitter"),
"tree-sitter-embedded-template": Label("@vendor__tree-sitter-embedded-template-0.23.2//:tree_sitter_embedded_template"),
"tree-sitter-ruby": Label("@vendor__tree-sitter-ruby-0.23.1//:tree_sitter_ruby"),
},
},
"rust/ast-generator": {
_COMMON_CONDITION: {
"anyhow": Label("@vendor__anyhow-1.0.95//:anyhow"),
"either": Label("@vendor__either-1.13.0//:either"),
"itertools": Label("@vendor__itertools-0.14.0//:itertools"),
"mustache": Label("@vendor__mustache-0.9.0//:mustache"),
"proc-macro2": Label("@vendor__proc-macro2-1.0.92//:proc_macro2"),
"quote": Label("@vendor__quote-1.0.38//:quote"),
"serde": Label("@vendor__serde-1.0.217//:serde"),
"stdx": Label("@vendor__ra_ap_stdx-0.0.248//:ra_ap_stdx"),
"ungrammar": Label("@vendor__ungrammar-1.16.1//:ungrammar"),
},
@@ -373,7 +376,7 @@ _NORMAL_DEPENDENCIES = {
"serde_json": Label("@vendor__serde_json-1.0.135//:serde_json"),
"tracing": Label("@vendor__tracing-0.1.41//:tracing"),
"tracing-subscriber": Label("@vendor__tracing-subscriber-0.3.19//:tracing_subscriber"),
"tree-sitter": Label("@vendor__tree-sitter-0.24.5//:tree_sitter"),
"tree-sitter": Label("@vendor__tree-sitter-0.24.6//:tree_sitter"),
},
},
}
@@ -829,12 +832,12 @@ def crate_repositories():
maybe(
http_archive,
name = "vendor__bstr-1.11.1",
sha256 = "786a307d683a5bf92e6fd5fd69a7eb613751668d1d8d67d802846dfe367c62c8",
name = "vendor__bstr-1.11.3",
sha256 = "531a9155a481e2ee699d4f98f43c0ca4ff8ee1bfd55c31e9e98fb29d2b176fe0",
type = "tar.gz",
urls = ["https://static.crates.io/crates/bstr/1.11.1/download"],
strip_prefix = "bstr-1.11.1",
build_file = Label("//misc/bazel/3rdparty/tree_sitter_extractors_deps:BUILD.bstr-1.11.1.bazel"),
urls = ["https://static.crates.io/crates/bstr/1.11.3/download"],
strip_prefix = "bstr-1.11.3",
build_file = Label("//misc/bazel/3rdparty/tree_sitter_extractors_deps:BUILD.bstr-1.11.3.bazel"),
)
maybe(
@@ -849,12 +852,12 @@ def crate_repositories():
maybe(
http_archive,
name = "vendor__bytemuck-1.20.0",
sha256 = "8b37c88a63ffd85d15b406896cc343916d7cf57838a847b3a6f2ca5d39a5695a",
name = "vendor__bytemuck-1.21.0",
sha256 = "ef657dfab802224e671f5818e9a4935f9b1957ed18e58292690cc39e7a4092a3",
type = "tar.gz",
urls = ["https://static.crates.io/crates/bytemuck/1.20.0/download"],
strip_prefix = "bytemuck-1.20.0",
build_file = Label("//misc/bazel/3rdparty/tree_sitter_extractors_deps:BUILD.bytemuck-1.20.0.bazel"),
urls = ["https://static.crates.io/crates/bytemuck/1.21.0/download"],
strip_prefix = "bytemuck-1.21.0",
build_file = Label("//misc/bazel/3rdparty/tree_sitter_extractors_deps:BUILD.bytemuck-1.21.0.bazel"),
)
maybe(
@@ -1707,6 +1710,16 @@ def crate_repositories():
build_file = Label("//misc/bazel/3rdparty/tree_sitter_extractors_deps:BUILD.lock_api-0.4.12.bazel"),
)
maybe(
http_archive,
name = "vendor__log-0.3.9",
sha256 = "e19e8d5c34a3e0e2223db8e060f9e8264aeeb5c5fc64a4ee9965c062211c024b",
type = "tar.gz",
urls = ["https://static.crates.io/crates/log/0.3.9/download"],
strip_prefix = "log-0.3.9",
build_file = Label("//misc/bazel/3rdparty/tree_sitter_extractors_deps:BUILD.log-0.3.9.bazel"),
)
maybe(
http_archive,
name = "vendor__log-0.4.22",
@@ -1787,6 +1800,16 @@ def crate_repositories():
build_file = Label("//misc/bazel/3rdparty/tree_sitter_extractors_deps:BUILD.miow-0.6.0.bazel"),
)
maybe(
http_archive,
name = "vendor__mustache-0.9.0",
sha256 = "51956ef1c5d20a1384524d91e616fb44dfc7d8f249bf696d49c97dd3289ecab5",
type = "tar.gz",
urls = ["https://static.crates.io/crates/mustache/0.9.0/download"],
strip_prefix = "mustache-0.9.0",
build_file = Label("//misc/bazel/3rdparty/tree_sitter_extractors_deps:BUILD.mustache-0.9.0.bazel"),
)
maybe(
http_archive,
name = "vendor__nohash-hasher-0.2.0",
@@ -2838,12 +2861,12 @@ def crate_repositories():
maybe(
http_archive,
name = "vendor__tree-sitter-0.24.5",
sha256 = "8ac95b18f0f727aaaa012bd5179a1916706ee3ed071920fdbda738750b0c0bf5",
name = "vendor__tree-sitter-0.24.6",
sha256 = "5f2434c86ba59ed15af56039cc5bf1acf8ba76ce301e32ef08827388ef285ec5",
type = "tar.gz",
urls = ["https://static.crates.io/crates/tree-sitter/0.24.5/download"],
strip_prefix = "tree-sitter-0.24.5",
build_file = Label("//misc/bazel/3rdparty/tree_sitter_extractors_deps:BUILD.tree-sitter-0.24.5.bazel"),
urls = ["https://static.crates.io/crates/tree-sitter/0.24.6/download"],
strip_prefix = "tree-sitter-0.24.6",
build_file = Label("//misc/bazel/3rdparty/tree_sitter_extractors_deps:BUILD.tree-sitter-0.24.6.bazel"),
)
maybe(
@@ -3371,6 +3394,7 @@ def crate_repositories():
struct(repo = "vendor__itertools-0.14.0", is_dev_dep = False),
struct(repo = "vendor__lazy_static-1.5.0", is_dev_dep = False),
struct(repo = "vendor__log-0.4.22", is_dev_dep = False),
struct(repo = "vendor__mustache-0.9.0", is_dev_dep = False),
struct(repo = "vendor__num-traits-0.2.19", is_dev_dep = False),
struct(repo = "vendor__num_cpus-1.16.0", is_dev_dep = False),
struct(repo = "vendor__proc-macro2-1.0.92", is_dev_dep = False),
@@ -3399,7 +3423,7 @@ def crate_repositories():
struct(repo = "vendor__syn-2.0.95", is_dev_dep = False),
struct(repo = "vendor__tracing-0.1.41", is_dev_dep = False),
struct(repo = "vendor__tracing-subscriber-0.3.19", is_dev_dep = False),
struct(repo = "vendor__tree-sitter-0.24.5", is_dev_dep = False),
struct(repo = "vendor__tree-sitter-0.24.6", is_dev_dep = False),
struct(repo = "vendor__tree-sitter-embedded-template-0.23.2", is_dev_dep = False),
struct(repo = "vendor__tree-sitter-ruby-0.23.1", is_dev_dep = False),
struct(repo = "vendor__triomphe-0.1.14", is_dev_dep = False),

1
rust/ast-generator/BUILD.bazel
View File

@@ -56,6 +56,7 @@ codeql_rust_binary(
) + [":codegen"],
aliases = aliases(),
args = ["$(rlocationpath :ungram)"],
compile_data = glob(["src/templates/*.mustache"]),
data = [":ungram"],
proc_macro_deps = all_crate_deps(
proc_macro = True,

3
rust/ast-generator/Cargo.toml
View File

@@ -11,3 +11,6 @@ quote = "1.0.38"
either = "1.13.0"
stdx = {package = "ra_ap_stdx", version = "0.0.248"}
itertools = "0.14.0"
mustache = "0.9.0"
serde = { version = "1.0.217", features = ["derive"] }
anyhow = "1.0.95"

587
rust/ast-generator/src/main.rs
View File

@@ -1,9 +1,11 @@
use std::io::Write;
use std::{fs, path::PathBuf};
pub mod codegen;
mod flags;
use crate::codegen::grammar::ast_src::{AstEnumSrc, Cardinality};
use codegen::grammar::ast_src::{AstNodeSrc, AstSrc, Field};
use itertools::Itertools;
use serde::Serialize;
use std::collections::{BTreeMap, BTreeSet};
use std::env;
use ungrammar::Grammar;
@@ -56,92 +58,116 @@ fn to_lower_snake_case(s: &str) -> String {
buf
}
#[derive(Serialize)]
struct SchemaField {
name: String,
ty: String,
child: bool,
}
#[derive(Serialize)]
struct SchemaClass {
name: String,
bases: Vec<String>,
fields: Vec<SchemaField>,
}
#[derive(Serialize, Default)]
struct Schema {
classes: Vec<SchemaClass>,
}
fn get_bases(name: &str, super_types: &BTreeMap<String, BTreeSet<String>>) -> Vec<String> {
super_types
.get(name)
.map(|tys| tys.iter().map(|t| class_name(t)).collect())
.unwrap_or_else(|| vec!["AstNode".to_string()])
}
fn enum_src_to_schema_class(
node: &AstEnumSrc,
super_types: &BTreeMap<String, BTreeSet<String>>,
) -> SchemaClass {
SchemaClass {
name: class_name(&node.name),
bases: get_bases(&node.name, super_types),
fields: Vec::new(),
}
}
fn node_src_to_schema_class(
node: &AstNodeSrc,
super_types: &BTreeMap<String, BTreeSet<String>>,
) -> SchemaClass {
SchemaClass {
name: class_name(&node.name),
bases: get_bases(&node.name, super_types),
fields: get_fields(node)
.iter()
.map(|f| {
let (ty, child) = match &f.ty {
FieldType::String => ("optional[string]".to_string(), false),
FieldType::Predicate => ("predicate".to_string(), false),
FieldType::Optional(ty) => (format!("optional[\"{}\"]", class_name(ty)), true),
FieldType::List(ty) => (format!("list[\"{}\"]", class_name(ty)), true),
};
SchemaField {
name: property_name(&node.name, &f.name),
ty,
child,
}
})
.collect(),
}
}
fn fix_blank_lines(s: &str) -> String {
// mustache is not very good at avoiding blank lines
// adopting the workaround from https://github.com/groue/GRMustache/issues/46#issuecomment-19498046
s.split("\n")
.filter(|line| !line.trim().is_empty())
.map(|line| if line == "" { "" } else { line })
.join("\n")
+ "\n"
}
fn write_schema(
grammar: &AstSrc,
super_types: BTreeMap<String, BTreeSet<String>>,
) -> std::io::Result<String> {
let mut buf: Vec<u8> = Vec::new();
writeln!(
buf,
"# Generated by `ast-generator`, do not edit by hand.\n"
)?;
writeln!(buf, "from .prelude import *")?;
) -> mustache::Result<String> {
let mut schema = Schema::default();
schema.classes.extend(
grammar
.enums
.iter()
.map(|node| enum_src_to_schema_class(node, &super_types)),
);
schema.classes.extend(
grammar
.nodes
.iter()
.map(|node| node_src_to_schema_class(node, &super_types)),
);
// the concat dance is currently required by bazel
let template = mustache::compile_str(include_str!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/src/templates/schema.mustache"
)))?;
let res = template.render_to_string(&schema)?;
Ok(fix_blank_lines(&res))
}
for node in &grammar.enums {
let super_classses = if let Some(cls) = super_types.get(&node.name) {
let super_classes: Vec<String> = cls.iter().map(|s| class_name(s)).collect();
super_classes.join(",")
} else {
"AstNode".to_owned()
};
writeln!(
buf,
"\nclass {}({}):",
class_name(&node.name),
super_classses
)?;
writeln!(buf, " pass")?;
}
for node in &grammar.nodes {
let super_classses = if let Some(cls) = super_types.get(&node.name) {
let super_classes: Vec<String> = cls.iter().map(|s| class_name(s)).collect();
super_classes.join(",")
} else {
"AstNode".to_owned()
};
writeln!(
buf,
"\nclass {}({}):",
class_name(&node.name),
super_classses
)?;
let mut empty = true;
for field in get_fields(node) {
if field.tp == "SyntaxToken" {
continue;
}
empty = false;
if field.tp == "predicate" {
writeln!(
buf,
" {}: predicate",
property_name(&node.name, &field.name),
)?;
} else if field.tp == "string" {
writeln!(
buf,
" {}: optional[string]",
property_name(&node.name, &field.name),
)?;
} else {
let list = field.is_many;
let (o, c) = if list {
("list[", "]")
} else {
("optional[", "]")
};
writeln!(
buf,
" {}: {}\"{}\"{} | child",
property_name(&node.name, &field.name),
o,
class_name(&field.tp),
c
)?;
};
}
if empty {
writeln!(buf, " pass")?;
}
}
Ok(String::from_utf8_lossy(&buf).to_string())
#[derive(Eq, PartialEq)]
enum FieldType {
String,
Predicate,
Optional(String),
List(String),
}
struct FieldInfo {
name: String,
tp: String,
is_many: bool,
ty: FieldType,
}
fn get_fields(node: &AstNodeSrc) -> Vec<FieldInfo> {
let mut result = Vec::new();
@@ -154,8 +180,7 @@ fn get_fields(node: &AstNodeSrc) -> Vec<FieldInfo> {
if predicates.contains(&name.as_str()) {
result.push(FieldInfo {
name: format!("is_{name}"),
tp: "predicate".to_string(),
is_many: false,
ty: FieldType::Predicate,
});
}
}
@@ -165,210 +190,177 @@ fn get_fields(node: &AstNodeSrc) -> Vec<FieldInfo> {
"Name" | "NameRef" | "Lifetime" => {
result.push(FieldInfo {
name: "text".to_string(),
tp: "string".to_string(),
is_many: false,
ty: FieldType::String,
});
}
"Abi" => {
result.push(FieldInfo {
name: "abi_string".to_string(),
tp: "string".to_string(),
is_many: false,
ty: FieldType::String,
});
}
"Literal" => {
result.push(FieldInfo {
name: "text_value".to_string(),
tp: "string".to_string(),
is_many: false,
ty: FieldType::String,
});
}
"PrefixExpr" => {
result.push(FieldInfo {
name: "operator_name".to_string(),
tp: "string".to_string(),
is_many: false,
ty: FieldType::String,
});
}
"BinExpr" => {
result.push(FieldInfo {
name: "lhs".to_string(),
tp: "Expr".to_string(),
is_many: false,
ty: FieldType::Optional("Expr".to_string()),
});
result.push(FieldInfo {
name: "rhs".to_string(),
tp: "Expr".to_string(),
is_many: false,
ty: FieldType::Optional("Expr".to_string()),
});
result.push(FieldInfo {
name: "operator_name".to_string(),
tp: "string".to_string(),
is_many: false,
ty: FieldType::String,
});
}
"IfExpr" => {
result.push(FieldInfo {
name: "then_branch".to_string(),
tp: "BlockExpr".to_string(),
is_many: false,
ty: FieldType::Optional("BlockExpr".to_string()),
});
result.push(FieldInfo {
name: "else_branch".to_string(),
tp: "ElseBranch".to_string(),
is_many: false,
ty: FieldType::Optional("ElseBranch".to_string()),
});
result.push(FieldInfo {
name: "condition".to_string(),
tp: "Expr".to_string(),
is_many: false,
ty: FieldType::Optional("Expr".to_string()),
});
}
"RangeExpr" => {
result.push(FieldInfo {
name: "start".to_string(),
tp: "Expr".to_string(),
is_many: false,
ty: FieldType::Optional("Expr".to_string()),
});
result.push(FieldInfo {
name: "end".to_string(),
tp: "Expr".to_string(),
is_many: false,
ty: FieldType::Optional("Expr".to_string()),
});
result.push(FieldInfo {
name: "operator_name".to_string(),
tp: "string".to_string(),
is_many: false,
ty: FieldType::String,
});
}
"RangePat" => {
result.push(FieldInfo {
name: "start".to_string(),
tp: "Pat".to_string(),
is_many: false,
ty: FieldType::Optional("Pat".to_string()),
});
result.push(FieldInfo {
name: "end".to_string(),
tp: "Pat".to_string(),
is_many: false,
ty: FieldType::Optional("Pat".to_string()),
});
result.push(FieldInfo {
name: "operator_name".to_string(),
tp: "string".to_string(),
is_many: false,
ty: FieldType::String,
});
}
"IndexExpr" => {
result.push(FieldInfo {
name: "index".to_string(),
tp: "Expr".to_string(),
is_many: false,
ty: FieldType::Optional("Expr".to_string()),
});
result.push(FieldInfo {
name: "base".to_string(),
tp: "Expr".to_string(),
is_many: false,
ty: FieldType::Optional("Expr".to_string()),
});
}
"Impl" => {
result.push(FieldInfo {
name: "trait_".to_string(),
tp: "Type".to_string(),
is_many: false,
ty: FieldType::Optional("Type".to_string()),
});
result.push(FieldInfo {
name: "self_ty".to_string(),
tp: "Type".to_string(),
is_many: false,
ty: FieldType::Optional("Type".to_string()),
});
}
"ForExpr" => {
result.push(FieldInfo {
name: "iterable".to_string(),
tp: "Expr".to_string(),
is_many: false,
ty: FieldType::Optional("Expr".to_string()),
});
}
"WhileExpr" => {
result.push(FieldInfo {
name: "condition".to_string(),
tp: "Expr".to_string(),
is_many: false,
ty: FieldType::Optional("Expr".to_string()),
});
}
"MatchGuard" => {
result.push(FieldInfo {
name: "condition".to_string(),
tp: "Expr".to_string(),
is_many: false,
ty: FieldType::Optional("Expr".to_string()),
});
}
"MacroDef" => {
result.push(FieldInfo {
name: "args".to_string(),
tp: "TokenTree".to_string(),
is_many: false,
ty: FieldType::Optional("TokenTree".to_string()),
});
result.push(FieldInfo {
name: "body".to_string(),
tp: "TokenTree".to_string(),
is_many: false,
ty: FieldType::Optional("TokenTree".to_string()),
});
}
"FormatArgsExpr" => {
result.push(FieldInfo {
name: "args".to_string(),
tp: "FormatArgsArg".to_string(),
is_many: true,
ty: FieldType::List("FormatArgsArg".to_string()),
});
}
"ArgList" => {
result.push(FieldInfo {
name: "args".to_string(),
tp: "Expr".to_string(),
is_many: true,
ty: FieldType::List("Expr".to_string()),
});
}
"Fn" => {
result.push(FieldInfo {
name: "body".to_string(),
tp: "BlockExpr".to_string(),
is_many: false,
ty: FieldType::Optional("BlockExpr".to_string()),
});
}
"Const" => {
result.push(FieldInfo {
name: "body".to_string(),
tp: "Expr".to_string(),
is_many: false,
ty: FieldType::Optional("Expr".to_string()),
});
}
"Static" => {
result.push(FieldInfo {
name: "body".to_string(),
tp: "Expr".to_string(),
is_many: false,
ty: FieldType::Optional("Expr".to_string()),
});
}
"ClosureExpr" => {
result.push(FieldInfo {
name: "body".to_string(),
tp: "Expr".to_string(),
is_many: false,
ty: FieldType::Optional("Expr".to_string()),
});
}
"ArrayExpr" => {
result.push(FieldInfo {
name: "is_semicolon".to_string(),
tp: "predicate".to_string(),
is_many: false,
ty: FieldType::Predicate,
});
}
"SelfParam" => {
result.push(FieldInfo {
name: "is_amp".to_string(),
tp: "predicate".to_string(),
is_many: false,
ty: FieldType::Predicate,
});
}
_ => {}
@@ -379,72 +371,69 @@ fn get_fields(node: &AstNodeSrc) -> Vec<FieldInfo> {
if node.name == "ArrayExpr" && field.method_name() == "expr" {
continue;
}
let ty = match field {
Field::Token(_) => continue,
Field::Node {
ty, cardinality, ..
} => match cardinality {
Cardinality::Optional => FieldType::Optional(ty.clone()),
Cardinality::Many => FieldType::List(ty.clone()),
},
};
result.push(FieldInfo {
name: field.method_name(),
tp: field.ty().to_string(),
is_many: field.is_many(),
ty,
});
}
for trait_ in &node.traits {
match trait_.as_str() {
"HasAttrs" => result.push(FieldInfo {
name: "attrs".to_owned(),
tp: "Attr".to_owned(),
is_many: true,
ty: FieldType::List("Attr".to_owned()),
}),
"HasName" => result.push(FieldInfo {
name: "name".to_owned(),
tp: "Name".to_owned(),
is_many: false,
ty: FieldType::Optional("Name".to_owned()),
}),
"HasVisibility" => result.push(FieldInfo {
name: "visibility".to_owned(),
tp: "Visibility".to_owned(),
is_many: false,
ty: FieldType::Optional("Visibility".to_owned()),
}),
"HasGenericParams" => {
result.push(FieldInfo {
name: "generic_param_list".to_owned(),
tp: "GenericParamList".to_owned(),
is_many: false,
ty: FieldType::Optional("GenericParamList".to_owned()),
});
result.push(FieldInfo {
name: "where_clause".to_owned(),
tp: "WhereClause".to_owned(),
is_many: false,
ty: FieldType::Optional("WhereClause".to_owned()),
})
}
"HasGenericArgs" => result.push(FieldInfo {
name: "generic_arg_list".to_owned(),
tp: "GenericArgList".to_owned(),
is_many: false,
ty: FieldType::Optional("GenericArgList".to_owned()),
}),
"HasTypeBounds" => result.push(FieldInfo {
name: "type_bound_list".to_owned(),
tp: "TypeBoundList".to_owned(),
is_many: false,
ty: FieldType::Optional("TypeBoundList".to_owned()),
}),
"HasModuleItem" => result.push(FieldInfo {
name: "items".to_owned(),
tp: "Item".to_owned(),
is_many: true,
ty: FieldType::List("Item".to_owned()),
}),
"HasLoopBody" => {
result.push(FieldInfo {
name: "label".to_owned(),
tp: "Label".to_owned(),
is_many: false,
ty: FieldType::Optional("Label".to_owned()),
});
result.push(FieldInfo {
name: "loop_body".to_owned(),
tp: "BlockExpr".to_owned(),
is_many: false,
ty: FieldType::Optional("BlockExpr".to_owned()),
})
}
"HasArgList" => result.push(FieldInfo {
name: "arg_list".to_owned(),
tp: "ArgList".to_owned(),
is_many: false,
ty: FieldType::Optional("ArgList".to_owned()),
}),
"HasDocComments" => {}
@@ -455,150 +444,122 @@ fn get_fields(node: &AstNodeSrc) -> Vec<FieldInfo> {
result
}
fn write_extractor(grammar: &AstSrc) -> std::io::Result<String> {
let mut buf: Vec<u8> = Vec::new();
writeln!(
buf,
"//! Generated by `ast-generator`, do not edit by hand.\n
#![cfg_attr(any(), rustfmt::skip)]
use super::base::Translator;
use super::mappings::TextValue;
use crate::emit_detached;
use crate::generated;
use crate::trap::{{Label, TrapId}};
use ra_ap_syntax::ast::{{
HasArgList, HasAttrs, HasGenericArgs, HasGenericParams, HasLoopBody, HasModuleItem, HasName,
HasTypeBounds, HasVisibility, RangeItem,
}};
use ra_ap_syntax::{{ast, AstNode}};
impl Translator<'_> {{
fn emit_else_branch(&mut self, node: ast::ElseBranch) -> Option<Label<generated::Expr>> {{
match node {{
ast::ElseBranch::IfExpr(inner) => self.emit_if_expr(inner).map(Into::into),
ast::ElseBranch::Block(inner) => self.emit_block_expr(inner).map(Into::into),
}}
}}\n"
)?;
for node in &grammar.enums {
let type_name = &node.name;
let class_name = class_name(&node.name);
writeln!(
buf,
" pub(crate) fn emit_{}(&mut self, node: ast::{}) -> Option<Label<generated::{}>> {{",
to_lower_snake_case(type_name),
type_name,
class_name
)?;
writeln!(buf, " match node {{")?;
for variant in &node.variants {
writeln!(
buf,
" ast::{}::{}(inner) => self.emit_{}(inner).map(Into::into),",
type_name,
variant,
to_lower_snake_case(variant)
)?;
}
writeln!(buf, " }}")?;
writeln!(buf, " }}\n")?;
}
for node in &grammar.nodes {
let type_name = &node.name;
let class_name = class_name(&node.name);
writeln!(
buf,
" pub(crate) fn emit_{}(&mut self, node: ast::{}) -> Option<Label<generated::{}>> {{",
to_lower_snake_case(type_name),
type_name,
class_name
)?;
for field in get_fields(node) {
if &field.tp == "SyntaxToken" {
continue;
}
if field.name == "attrs" {
// special case: this means the node type implements `HasAttrs`, and we want to
// check whether it was not excluded by a `cfg` attribute
writeln!(
buf,
" if self.should_be_excluded(&node) {{ return None; }}"
)?;
}
let type_name = &field.tp;
let struct_field_name = &field.name;
let class_field_name = property_name(&node.name, &field.name);
if field.tp == "predicate" {
writeln!(
buf,
" let {} = node.{}_token().is_some();",
class_field_name,
&struct_field_name[3..],
)?;
} else if field.tp == "string" {
writeln!(
buf,
" let {} = node.try_get_text();",
class_field_name,
)?;
} else if field.is_many {
writeln!(
buf,
" let {} = node.{}().filter_map(|x| self.emit_{}(x)).collect();",
class_field_name,
struct_field_name,
to_lower_snake_case(type_name)
)?;
} else {
writeln!(
buf,
" let {} = node.{}().and_then(|x| self.emit_{}(x));",
class_field_name,
struct_field_name,
to_lower_snake_case(type_name)
)?;
}
}
writeln!(
buf,
" let label = self.trap.emit(generated::{} {{",
class_name
)?;
writeln!(buf, " id: TrapId::Star,")?;
for field in get_fields(node) {
if field.tp == "SyntaxToken" {
continue;
}
let class_field_name: String = property_name(&node.name, &field.name);
writeln!(buf, " {},", class_field_name)?;
}
writeln!(buf, " }});")?;
writeln!(buf, " self.emit_location(label, &node);")?;
writeln!(
buf,
" emit_detached!({}, self, node, label);",
class_name
)?;
writeln!(
buf,
" self.emit_tokens(&node, label.into(), node.syntax().children_with_tokens());"
)?;
writeln!(buf, " Some(label)")?;
writeln!(buf, " }}\n")?;
}
writeln!(buf, "}}")?;
Ok(String::from_utf8_lossy(&buf).into_owned())
#[derive(Serialize)]
struct EnumVariantInfo {
name: String,
snake_case_name: String,
}
fn main() -> std::io::Result<()> {
#[derive(Serialize)]
struct ExtractorEnumInfo {
name: String,
snake_case_name: String,
ast_name: String,
variants: Vec<EnumVariantInfo>,
}
#[derive(Serialize, Default)]
struct ExtractorNodeFieldInfo {
name: String,
method: String,
snake_case_ty: String,
string: bool,
predicate: bool,
optional: bool,
list: bool,
}
#[derive(Serialize)]
struct ExtractorNodeInfo {
name: String,
snake_case_name: String,
ast_name: String,
fields: Vec<ExtractorNodeFieldInfo>,
has_attrs: bool,
}
#[derive(Serialize)]
struct ExtractorInfo {
enums: Vec<ExtractorEnumInfo>,
nodes: Vec<ExtractorNodeInfo>,
}
fn enum_to_extractor_info(node: &AstEnumSrc) -> ExtractorEnumInfo {
ExtractorEnumInfo {
name: class_name(&node.name),
snake_case_name: to_lower_snake_case(&node.name),
ast_name: node.name.clone(),
variants: node
.variants
.iter()
.map(|v| EnumVariantInfo {
name: v.clone(),
snake_case_name: to_lower_snake_case(v),
})
.collect(),
}
}
fn field_info_to_extractor_info(node: &AstNodeSrc, field: &FieldInfo) -> ExtractorNodeFieldInfo {
let name = property_name(&node.name, &field.name);
match &field.ty {
FieldType::String => ExtractorNodeFieldInfo {
name,
string: true,
..Default::default()
},
FieldType::Predicate => ExtractorNodeFieldInfo {
name,
method: format!("{}_token", &field.name[3..]),
predicate: true,
..Default::default()
},
FieldType::Optional(ty) => ExtractorNodeFieldInfo {
name,
method: field.name.clone(),
snake_case_ty: to_lower_snake_case(ty),
optional: true,
..Default::default()
},
FieldType::List(ty) => ExtractorNodeFieldInfo {
name,
method: field.name.clone(),
snake_case_ty: to_lower_snake_case(ty),
list: true,
..Default::default()
},
}
}
fn node_to_extractor_info(node: &AstNodeSrc) -> ExtractorNodeInfo {
let fields = get_fields(node);
let has_attrs = fields.iter().any(|f| f.name == "attrs");
ExtractorNodeInfo {
name: class_name(&node.name),
snake_case_name: to_lower_snake_case(&node.name),
ast_name: node.name.clone(),
fields: fields
.iter()
.map(|f| field_info_to_extractor_info(node, f))
.collect(),
has_attrs,
}
}
fn write_extractor(grammar: &AstSrc) -> mustache::Result<String> {
let extractor_info = ExtractorInfo {
enums: grammar.enums.iter().map(enum_to_extractor_info).collect(),
nodes: grammar.nodes.iter().map(node_to_extractor_info).collect(),
};
// the concat dance is currently required by bazel
let template = mustache::compile_str(include_str!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/src/templates/extractor.mustache"
)))?;
let res = template.render_to_string(&extractor_info)?;
Ok(fix_blank_lines(&res))
}
fn main() -> anyhow::Result<()> {
let grammar = PathBuf::from("..").join(env::args().nth(1).expect("grammar file path required"));
let grammar: Grammar = fs::read_to_string(&grammar)
.unwrap_or_else(|_| panic!("Failed to parse grammar file: {}", grammar.display()))

65
rust/ast-generator/src/templates/extractor.mustache Normal file
View File

@@ -0,0 +1,65 @@
//! Generated by `ast-generator`, do not edit by hand.
¶{{! <- denotes empty line that should be kept, all blank lines are removed otherwise}}
#![cfg_attr(any(), rustfmt::skip)]
use super::base::Translator;
use super::mappings::TextValue;
use crate::emit_detached;
use crate::generated;
use crate::trap::{Label, TrapId};
use ra_ap_syntax::ast::{
HasArgList, HasAttrs, HasGenericArgs, HasGenericParams, HasLoopBody, HasModuleItem, HasName,
HasTypeBounds, HasVisibility, RangeItem,
};
use ra_ap_syntax::{ast, AstNode};
impl Translator<'_> {
fn emit_else_branch(&mut self, node: ast::ElseBranch) -> Option<Label<generated::Expr>> {
match node {
ast::ElseBranch::IfExpr(inner) => self.emit_if_expr(inner).map(Into::into),
ast::ElseBranch::Block(inner) => self.emit_block_expr(inner).map(Into::into),
}
}
{{#enums}}
pub(crate) fn emit_{{snake_case_name}}(&mut self, node: ast::{{ast_name}}) -> Option<Label<generated::{{name}}>> {
match node {
{{#variants}}
ast::{{ast_name}}::{{name}}(inner) => self.emit_{{snake_case_name}}(inner).map(Into::into),
{{/variants}}
}
}
{{/enums}}
{{#nodes}}
pub(crate) fn emit_{{snake_case_name}}(&mut self, node: ast::{{ast_name}}) -> Option<Label<generated::{{name}}>> {
{{#has_attrs}}
if self.should_be_excluded(&node) { return None; }
{{/has_attrs}}
{{#fields}}
{{#predicate}}
let {{name}} = node.{{method}}().is_some();
{{/predicate}}
{{#string}}
let {{name}} = node.try_get_text();
{{/string}}
{{#list}}
let {{name}} = node.{{method}}().filter_map(|x| self.emit_{{snake_case_ty}}(x)).collect();
{{/list}}
{{#optional}}
let {{name}} = node.{{method}}().and_then(|x| self.emit_{{snake_case_ty}}(x));
{{/optional}}
{{/fields}}
let label = self.trap.emit(generated::{{name}} {
id: TrapId::Star,
{{#fields}}
{{name}},
{{/fields}}
});
self.emit_location(label, &node);
emit_detached!({{name}}, self, node, label);
self.emit_tokens(&node, label.into(), node.syntax().children_with_tokens());
Some(label)
}
{{/nodes}}
}

13
rust/ast-generator/src/templates/schema.mustache Normal file
View File

@@ -0,0 +1,13 @@
# Generated by `ast-generator`, do not edit by hand.
¶{{! <- denotes empty line that should be kept, all blank lines are removed otherwise}}
from .prelude import *
{{#classes}}
class {{name}}({{#bases}}{{.}}, {{/bases}}):
{{#fields}}
{{name}}: {{{ty}}}{{#child}} | child{{/child}}
{{/fields}}
{{^fields}}
pass
{{/fields}}
{{/classes}}

23
rust/extractor/src/translate/generated.rs generated
View File

@@ -288,8 +288,8 @@ impl Translator<'_> {
}
pub(crate) fn emit_asm_expr(&mut self, node: ast::AsmExpr) -> Option<Label<generated::AsmExpr>> {
let asm_pieces = node.asm_pieces().filter_map(|x| self.emit_asm_piece(x)).collect();
if self.should_be_excluded(&node) { return None; }
let asm_pieces = node.asm_pieces().filter_map(|x| self.emit_asm_piece(x)).collect();
let attrs = node.attrs().filter_map(|x| self.emit_attr(x)).collect();
let template = node.template().filter_map(|x| self.emit_expr(x)).collect();
let label = self.trap.emit(generated::AsmExpr {
@@ -409,8 +409,8 @@ impl Translator<'_> {
}
pub(crate) fn emit_assoc_item_list(&mut self, node: ast::AssocItemList) -> Option<Label<generated::AssocItemList>> {
let assoc_items = node.assoc_items().filter_map(|x| self.emit_assoc_item(x)).collect();
if self.should_be_excluded(&node) { return None; }
let assoc_items = node.assoc_items().filter_map(|x| self.emit_assoc_item(x)).collect();
let attrs = node.attrs().filter_map(|x| self.emit_attr(x)).collect();
let label = self.trap.emit(generated::AssocItemList {
id: TrapId::Star,
@@ -569,8 +569,8 @@ impl Translator<'_> {
}
pub(crate) fn emit_call_expr(&mut self, node: ast::CallExpr) -> Option<Label<generated::CallExpr>> {
let arg_list = node.arg_list().and_then(|x| self.emit_arg_list(x));
if self.should_be_excluded(&node) { return None; }
let arg_list = node.arg_list().and_then(|x| self.emit_arg_list(x));
let attrs = node.attrs().filter_map(|x| self.emit_attr(x)).collect();
let function = node.expr().and_then(|x| self.emit_expr(x));
let label = self.trap.emit(generated::CallExpr {
@@ -780,8 +780,8 @@ impl Translator<'_> {
}
pub(crate) fn emit_extern_block(&mut self, node: ast::ExternBlock) -> Option<Label<generated::ExternBlock>> {
let abi = node.abi().and_then(|x| self.emit_abi(x));
if self.should_be_excluded(&node) { return None; }
let abi = node.abi().and_then(|x| self.emit_abi(x));
let attrs = node.attrs().filter_map(|x| self.emit_attr(x)).collect();
let extern_item_list = node.extern_item_list().and_then(|x| self.emit_extern_item_list(x));
let is_unsafe = node.unsafe_token().is_some();
@@ -850,8 +850,8 @@ impl Translator<'_> {
}
pub(crate) fn emit_fn(&mut self, node: ast::Fn) -> Option<Label<generated::Function>> {
let abi = node.abi().and_then(|x| self.emit_abi(x));
if self.should_be_excluded(&node) { return None; }
let abi = node.abi().and_then(|x| self.emit_abi(x));
let attrs = node.attrs().filter_map(|x| self.emit_attr(x)).collect();
let body = node.body().and_then(|x| self.emit_block_expr(x));
let generic_param_list = node.generic_param_list().and_then(|x| self.emit_generic_param_list(x));
@@ -960,8 +960,8 @@ impl Translator<'_> {
}
pub(crate) fn emit_format_args_expr(&mut self, node: ast::FormatArgsExpr) -> Option<Label<generated::FormatArgsExpr>> {
let args = node.args().filter_map(|x| self.emit_format_args_arg(x)).collect();
if self.should_be_excluded(&node) { return None; }
let args = node.args().filter_map(|x| self.emit_format_args_arg(x)).collect();
let attrs = node.attrs().filter_map(|x| self.emit_attr(x)).collect();
let template = node.template().and_then(|x| self.emit_expr(x));
let label = self.trap.emit(generated::FormatArgsExpr {
@@ -1041,8 +1041,8 @@ impl Translator<'_> {
}
pub(crate) fn emit_impl(&mut self, node: ast::Impl) -> Option<Label<generated::Impl>> {
let assoc_item_list = node.assoc_item_list().and_then(|x| self.emit_assoc_item_list(x));
if self.should_be_excluded(&node) { return None; }
let assoc_item_list = node.assoc_item_list().and_then(|x| self.emit_assoc_item_list(x));
let attrs = node.attrs().filter_map(|x| self.emit_attr(x)).collect();
let generic_param_list = node.generic_param_list().and_then(|x| self.emit_generic_param_list(x));
let is_const = node.const_token().is_some();
@@ -1290,8 +1290,8 @@ impl Translator<'_> {
}
pub(crate) fn emit_macro_def(&mut self, node: ast::MacroDef) -> Option<Label<generated::MacroDef>> {
let args = node.args().and_then(|x| self.emit_token_tree(x));
if self.should_be_excluded(&node) { return None; }
let args = node.args().and_then(|x| self.emit_token_tree(x));
let attrs = node.attrs().filter_map(|x| self.emit_attr(x)).collect();
let body = node.body().and_then(|x| self.emit_token_tree(x));
let name = node.name().and_then(|x| self.emit_name(x));
@@ -1411,8 +1411,8 @@ impl Translator<'_> {
}
pub(crate) fn emit_match_arm_list(&mut self, node: ast::MatchArmList) -> Option<Label<generated::MatchArmList>> {
let arms = node.arms().filter_map(|x| self.emit_match_arm(x)).collect();
if self.should_be_excluded(&node) { return None; }
let arms = node.arms().filter_map(|x| self.emit_match_arm(x)).collect();
let attrs = node.attrs().filter_map(|x| self.emit_attr(x)).collect();
let label = self.trap.emit(generated::MatchArmList {
id: TrapId::Star,
@@ -1473,8 +1473,8 @@ impl Translator<'_> {
}
pub(crate) fn emit_method_call_expr(&mut self, node: ast::MethodCallExpr) -> Option<Label<generated::MethodCallExpr>> {
let arg_list = node.arg_list().and_then(|x| self.emit_arg_list(x));
if self.should_be_excluded(&node) { return None; }
let arg_list = node.arg_list().and_then(|x| self.emit_arg_list(x));
let attrs = node.attrs().filter_map(|x| self.emit_attr(x)).collect();
let generic_arg_list = node.generic_arg_list().and_then(|x| self.emit_generic_arg_list(x));
let name_ref = node.name_ref().and_then(|x| self.emit_name_ref(x));
@@ -2179,8 +2179,8 @@ impl Translator<'_> {
}
pub(crate) fn emit_trait(&mut self, node: ast::Trait) -> Option<Label<generated::Trait>> {
let assoc_item_list = node.assoc_item_list().and_then(|x| self.emit_assoc_item_list(x));
if self.should_be_excluded(&node) { return None; }
let assoc_item_list = node.assoc_item_list().and_then(|x| self.emit_assoc_item_list(x));
let attrs = node.attrs().filter_map(|x| self.emit_attr(x)).collect();
let generic_param_list = node.generic_param_list().and_then(|x| self.emit_generic_param_list(x));
let is_auto = node.auto_token().is_some();
@@ -2643,5 +2643,4 @@ impl Translator<'_> {
self.emit_tokens(&node, label.into(), node.syntax().children_with_tokens());
Some(label)
}
}

1500
rust/schema/ast.py generated
View File

File diff suppressed because it is too large Load Diff