hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2025-12-16 16:53:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

Rust: make ast-generator use mustache templates

Browse Source 
This simplifies the code and decouples the code template from the data
that is fed into it.
This commit is contained in:
Paolo Tranquilli
2025-01-07 18:05:02 +01:00
parent b4811906ea
commit b390fac105
19 changed files with 1387 additions and 1120 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
Cargo.lock generated
View File

@@ -92,9 +92,9 @@ dependencies = [
[[package]]
name = "anyhow"
version = "1.0.94"
version = "1.0.95"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c1fd03a028ef38ba2276dce7e33fcd6369c158a1bca17946c4b1b701891c1ff7"
checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04"
[[package]]
name = "argfile"
@@ -116,11 +116,14 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
name = "ast-generator"
version = "0.1.0"
dependencies = [
"anyhow",
"either",
"itertools 0.12.1",
"mustache",
"proc-macro2",
"quote",
"ra_ap_stdx",
"serde",
"ungrammar",
]
@@ -412,7 +415,7 @@ dependencies = [
"figment",
"glob",
"itertools 0.13.0",
"log",
"log 0.4.22",
"num-traits",
"ra_ap_base_db",
"ra_ap_cfg",
@@ -585,7 +588,7 @@ version = "0.14.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d248bdd43ce613d87415282f69b9bb99d947d290b10962dd6c56233312c2ad5"
dependencies = [
"log",
"log 0.4.22",
]
[[package]]
@@ -755,7 +758,7 @@ checksum = "15f1ce686646e7f1e19bf7d5533fe443a45dbfb990e00629110797578b42fb19"
dependencies = [
"aho-corasick",
"bstr",
"log",
"log 0.4.22",
"regex-automata 0.4.9",
"regex-syntax 0.8.5",
]
@@ -1020,6 +1023,15 @@ dependencies = [
"scopeguard",
]
[[package]]
name = "log"
version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e19e8d5c34a3e0e2223db8e060f9e8264aeeb5c5fc64a4ee9965c062211c024b"
dependencies = [
"log 0.4.22",
]
[[package]]
name = "log"
version = "0.4.22"
@@ -1072,7 +1084,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c"
dependencies = [
"libc",
"log",
"log 0.4.22",
"wasi",
"windows-sys 0.48.0",
]
@@ -1086,6 +1098,16 @@ dependencies = [
"windows-sys 0.48.0",
]
[[package]]
name = "mustache"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "51956ef1c5d20a1384524d91e616fb44dfc7d8f249bf696d49c97dd3289ecab5"
dependencies = [
"log 0.3.9",
"serde",
]
[[package]]
name = "nohash-hasher"
version = "0.2.0"
@@ -1105,7 +1127,7 @@ dependencies = [
"inotify",
"kqueue",
"libc",
"log",
"log 0.4.22",
"mio",
"walkdir",
"windows-sys 0.48.0",
@@ -2068,9 +2090,9 @@ dependencies = [
[[package]]
name = "serde_with"
version = "3.11.0"
version = "3.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e28bdad6db2b8340e449f7108f020b3b092e8583a9e3fb82713e1d4e71fe817"
checksum = "d6b6f7f2fcb69f747921f79f3926bd1e203fce4fef62c268dd3abfb6d86029aa"
dependencies = [
"base64",
"chrono",
@@ -2086,9 +2108,9 @@ dependencies = [
[[package]]
name = "serde_with_macros"
version = "3.11.0"
version = "3.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9d846214a9854ef724f3da161b426242d8de7c1fc7de2f89bb1efcb154dca79d"
checksum = "8d00caa5193a3c8362ac2b73be6b9e768aa5a4b2f721d8f4b339600c3cb51f8e"
dependencies = [
"darling",
"proc-macro2",
@@ -2160,7 +2182,7 @@ checksum = "61c910772f992ab17d32d6760e167d2353f4130ed50e796752689556af07dc6b"
dependencies = [
"chrono",
"is-terminal",
"log",
"log 0.4.22",
"termcolor",
"thread_local",
]
@@ -2313,7 +2335,7 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3"
dependencies = [
"log",
"log 0.4.22",
"once_cell",
"tracing-core",
]
@@ -2502,7 +2524,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5f89bb38646b4f81674e8f5c3fb81b562be1fd936d84320f3264486418519c79"
dependencies = [
"bumpalo",
"log",
"log 0.4.22",
"proc-macro2",
"quote",
"syn",

2
MODULE.bazel
View File

@@ -70,7 +70,7 @@ use_repo(py_deps, "vendor__anyhow-1.0.44", "vendor__cc-1.0.70", "vendor__clap-2.
# deps for ruby+rust
# keep in sync by running `misc/bazel/3rdparty/update_cargo_deps.sh`
tree_sitter_extractors_deps = use_extension("//misc/bazel/3rdparty:tree_sitter_extractors_extension.bzl", "r")
use_repo(tree_sitter_extractors_deps, "vendor__anyhow-1.0.94", "vendor__argfile-0.2.1", "vendor__chrono-0.4.39", "vendor__clap-4.5.23", "vendor__dunce-1.0.5", "vendor__either-1.13.0", "vendor__encoding-0.2.33", "vendor__figment-0.10.19", "vendor__flate2-1.0.35", "vendor__glob-0.3.1", "vendor__globset-0.4.15", "vendor__itertools-0.12.1", "vendor__itertools-0.13.0", "vendor__lazy_static-1.5.0", "vendor__log-0.4.22", "vendor__num-traits-0.2.19", "vendor__num_cpus-1.16.0", "vendor__proc-macro2-1.0.92", "vendor__quote-1.0.37", "vendor__ra_ap_base_db-0.0.248", "vendor__ra_ap_cfg-0.0.248", "vendor__ra_ap_hir-0.0.248", "vendor__ra_ap_hir_def-0.0.248", "vendor__ra_ap_hir_expand-0.0.248", "vendor__ra_ap_ide_db-0.0.248", "vendor__ra_ap_intern-0.0.248", "vendor__ra_ap_load-cargo-0.0.248", "vendor__ra_ap_parser-0.0.248", "vendor__ra_ap_paths-0.0.248", "vendor__ra_ap_project_model-0.0.248", "vendor__ra_ap_span-0.0.248", "vendor__ra_ap_stdx-0.0.248", "vendor__ra_ap_syntax-0.0.248", "vendor__ra_ap_vfs-0.0.248", "vendor__rand-0.8.5", "vendor__rayon-1.10.0", "vendor__regex-1.11.1", "vendor__serde-1.0.216", "vendor__serde_json-1.0.133", "vendor__serde_with-3.11.0", "vendor__stderrlog-0.6.0", "vendor__syn-2.0.90", "vendor__tracing-0.1.41", "vendor__tracing-subscriber-0.3.19", "vendor__tree-sitter-0.24.5", "vendor__tree-sitter-embedded-template-0.23.2", "vendor__tree-sitter-json-0.24.8", "vendor__tree-sitter-ql-0.23.1", "vendor__tree-sitter-ruby-0.23.1", "vendor__triomphe-0.1.14", "vendor__ungrammar-1.16.1")
use_repo(tree_sitter_extractors_deps, "vendor__anyhow-1.0.95", "vendor__argfile-0.2.1", "vendor__chrono-0.4.39", "vendor__clap-4.5.23", "vendor__dunce-1.0.5", "vendor__either-1.13.0", "vendor__encoding-0.2.33", "vendor__figment-0.10.19", "vendor__flate2-1.0.35", "vendor__glob-0.3.1", "vendor__globset-0.4.15", "vendor__itertools-0.12.1", "vendor__itertools-0.13.0", "vendor__lazy_static-1.5.0", "vendor__log-0.4.22", "vendor__mustache-0.9.0", "vendor__num-traits-0.2.19", "vendor__num_cpus-1.16.0", "vendor__proc-macro2-1.0.92", "vendor__quote-1.0.37", "vendor__ra_ap_base_db-0.0.248", "vendor__ra_ap_cfg-0.0.248", "vendor__ra_ap_hir-0.0.248", "vendor__ra_ap_hir_def-0.0.248", "vendor__ra_ap_hir_expand-0.0.248", "vendor__ra_ap_ide_db-0.0.248", "vendor__ra_ap_intern-0.0.248", "vendor__ra_ap_load-cargo-0.0.248", "vendor__ra_ap_parser-0.0.248", "vendor__ra_ap_paths-0.0.248", "vendor__ra_ap_project_model-0.0.248", "vendor__ra_ap_span-0.0.248", "vendor__ra_ap_stdx-0.0.248", "vendor__ra_ap_syntax-0.0.248", "vendor__ra_ap_vfs-0.0.248", "vendor__rand-0.8.5", "vendor__rayon-1.10.0", "vendor__regex-1.11.1", "vendor__serde-1.0.216", "vendor__serde_json-1.0.133", "vendor__serde_with-3.12.0", "vendor__stderrlog-0.6.0", "vendor__syn-2.0.90", "vendor__tracing-0.1.41", "vendor__tracing-subscriber-0.3.19", "vendor__tree-sitter-0.24.5", "vendor__tree-sitter-embedded-template-0.23.2", "vendor__tree-sitter-json-0.24.8", "vendor__tree-sitter-ql-0.23.1", "vendor__tree-sitter-ruby-0.23.1", "vendor__triomphe-0.1.14", "vendor__ungrammar-1.16.1")
http_archive = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")

6
misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.anyhow-1.0.94.bazel → misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.anyhow-1.0.95.bazel generated vendored
View File

@@ -82,9 +82,9 @@ rust_library(
"@rules_rust//rust/platform:x86_64-unknown-none": [],
"//conditions:default": ["@platforms//:incompatible"],
}),
version = "1.0.94",
version = "1.0.95",
deps = [
"@vendor__anyhow-1.0.94//:build_script_build",
"@vendor__anyhow-1.0.95//:build_script_build",
],
)
@@ -137,7 +137,7 @@ cargo_build_script(
"noclippy",
"norustfmt",
],
version = "1.0.94",
version = "1.0.95",
visibility = ["//visibility:private"],
)

10
misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.bazel generated vendored
View File

@@ -33,7 +33,7 @@ filegroup(
# Workspace Member Dependencies
alias(
name = "anyhow",
actual = "@vendor__anyhow-1.0.94//:anyhow",
actual = "@vendor__anyhow-1.0.95//:anyhow",
tags = ["manual"],
)
@@ -121,6 +121,12 @@ alias(
tags = ["manual"],
)
alias(
name = "mustache",
actual = "@vendor__mustache-0.9.0//:mustache",
tags = ["manual"],
)
alias(
name = "num-traits",
actual = "@vendor__num-traits-0.2.19//:num_traits",
@@ -267,7 +273,7 @@ alias(
alias(
name = "serde_with",
actual = "@vendor__serde_with-3.11.0//:serde_with",
actual = "@vendor__serde_with-3.12.0//:serde_with",
tags = ["manual"],
)

88
misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.log-0.3.9.bazel generated vendored Normal file
View File

@@ -0,0 +1,88 @@
###############################################################################
# @generated
# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To
# regenerate this file, run the following:
#
# bazel run @@//misc/bazel/3rdparty:vendor_tree_sitter_extractors
###############################################################################
load("@rules_rust//rust:defs.bzl", "rust_library")
package(default_visibility = ["//visibility:public"])
rust_library(
name = "log",
srcs = glob(
include = ["**/*.rs"],
allow_empty = True,
),
compile_data = glob(
include = ["**"],
allow_empty = True,
exclude = [
"**/* *",
".tmp_git_root/**/*",
"BUILD",
"BUILD.bazel",
"WORKSPACE",
"WORKSPACE.bazel",
],
),
crate_features = [
"default",
"use_std",
],
crate_root = "src/lib.rs",
edition = "2015",
rustc_flags = [
"--cap-lints=allow",
],
tags = [
"cargo-bazel",
"crate-name=log",
"manual",
"noclippy",
"norustfmt",
],
target_compatible_with = select({
"@rules_rust//rust/platform:aarch64-apple-darwin": [],
"@rules_rust//rust/platform:aarch64-apple-ios": [],
"@rules_rust//rust/platform:aarch64-apple-ios-sim": [],
"@rules_rust//rust/platform:aarch64-fuchsia": [],
"@rules_rust//rust/platform:aarch64-linux-android": [],
"@rules_rust//rust/platform:aarch64-pc-windows-msvc": [],
"@rules_rust//rust/platform:aarch64-unknown-linux-gnu": [],
"@rules_rust//rust/platform:aarch64-unknown-nixos-gnu": [],
"@rules_rust//rust/platform:aarch64-unknown-nto-qnx710": [],
"@rules_rust//rust/platform:arm-unknown-linux-gnueabi": [],
"@rules_rust//rust/platform:armv7-linux-androideabi": [],
"@rules_rust//rust/platform:armv7-unknown-linux-gnueabi": [],
"@rules_rust//rust/platform:i686-apple-darwin": [],
"@rules_rust//rust/platform:i686-linux-android": [],
"@rules_rust//rust/platform:i686-pc-windows-msvc": [],
"@rules_rust//rust/platform:i686-unknown-freebsd": [],
"@rules_rust//rust/platform:i686-unknown-linux-gnu": [],
"@rules_rust//rust/platform:powerpc-unknown-linux-gnu": [],
"@rules_rust//rust/platform:riscv32imc-unknown-none-elf": [],
"@rules_rust//rust/platform:riscv64gc-unknown-none-elf": [],
"@rules_rust//rust/platform:s390x-unknown-linux-gnu": [],
"@rules_rust//rust/platform:thumbv7em-none-eabi": [],
"@rules_rust//rust/platform:thumbv8m.main-none-eabi": [],
"@rules_rust//rust/platform:wasm32-unknown-unknown": [],
"@rules_rust//rust/platform:wasm32-wasi": [],
"@rules_rust//rust/platform:x86_64-apple-darwin": [],
"@rules_rust//rust/platform:x86_64-apple-ios": [],
"@rules_rust//rust/platform:x86_64-fuchsia": [],
"@rules_rust//rust/platform:x86_64-linux-android": [],
"@rules_rust//rust/platform:x86_64-pc-windows-msvc": [],
"@rules_rust//rust/platform:x86_64-unknown-freebsd": [],
"@rules_rust//rust/platform:x86_64-unknown-linux-gnu": [],
"@rules_rust//rust/platform:x86_64-unknown-nixos-gnu": [],
"@rules_rust//rust/platform:x86_64-unknown-none": [],
"//conditions:default": ["@platforms//:incompatible"],
}),
version = "0.3.9",
deps = [
"@vendor__log-0.4.22//:log",
],
)

85
misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.mustache-0.9.0.bazel generated vendored Normal file
View File

@@ -0,0 +1,85 @@
###############################################################################
# @generated
# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To
# regenerate this file, run the following:
#
# bazel run @@//misc/bazel/3rdparty:vendor_tree_sitter_extractors
###############################################################################
load("@rules_rust//rust:defs.bzl", "rust_library")
package(default_visibility = ["//visibility:public"])
rust_library(
name = "mustache",
srcs = glob(
include = ["**/*.rs"],
allow_empty = True,
),
compile_data = glob(
include = ["**"],
allow_empty = True,
exclude = [
"**/* *",
".tmp_git_root/**/*",
"BUILD",
"BUILD.bazel",
"WORKSPACE",
"WORKSPACE.bazel",
],
),
crate_root = "src/lib.rs",
edition = "2015",
rustc_flags = [
"--cap-lints=allow",
],
tags = [
"cargo-bazel",
"crate-name=mustache",
"manual",
"noclippy",
"norustfmt",
],
target_compatible_with = select({
"@rules_rust//rust/platform:aarch64-apple-darwin": [],
"@rules_rust//rust/platform:aarch64-apple-ios": [],
"@rules_rust//rust/platform:aarch64-apple-ios-sim": [],
"@rules_rust//rust/platform:aarch64-fuchsia": [],
"@rules_rust//rust/platform:aarch64-linux-android": [],
"@rules_rust//rust/platform:aarch64-pc-windows-msvc": [],
"@rules_rust//rust/platform:aarch64-unknown-linux-gnu": [],
"@rules_rust//rust/platform:aarch64-unknown-nixos-gnu": [],
"@rules_rust//rust/platform:aarch64-unknown-nto-qnx710": [],
"@rules_rust//rust/platform:arm-unknown-linux-gnueabi": [],
"@rules_rust//rust/platform:armv7-linux-androideabi": [],
"@rules_rust//rust/platform:armv7-unknown-linux-gnueabi": [],
"@rules_rust//rust/platform:i686-apple-darwin": [],
"@rules_rust//rust/platform:i686-linux-android": [],
"@rules_rust//rust/platform:i686-pc-windows-msvc": [],
"@rules_rust//rust/platform:i686-unknown-freebsd": [],
"@rules_rust//rust/platform:i686-unknown-linux-gnu": [],
"@rules_rust//rust/platform:powerpc-unknown-linux-gnu": [],
"@rules_rust//rust/platform:riscv32imc-unknown-none-elf": [],
"@rules_rust//rust/platform:riscv64gc-unknown-none-elf": [],
"@rules_rust//rust/platform:s390x-unknown-linux-gnu": [],
"@rules_rust//rust/platform:thumbv7em-none-eabi": [],
"@rules_rust//rust/platform:thumbv8m.main-none-eabi": [],
"@rules_rust//rust/platform:wasm32-unknown-unknown": [],
"@rules_rust//rust/platform:wasm32-wasi": [],
"@rules_rust//rust/platform:x86_64-apple-darwin": [],
"@rules_rust//rust/platform:x86_64-apple-ios": [],
"@rules_rust//rust/platform:x86_64-fuchsia": [],
"@rules_rust//rust/platform:x86_64-linux-android": [],
"@rules_rust//rust/platform:x86_64-pc-windows-msvc": [],
"@rules_rust//rust/platform:x86_64-unknown-freebsd": [],
"@rules_rust//rust/platform:x86_64-unknown-linux-gnu": [],
"@rules_rust//rust/platform:x86_64-unknown-nixos-gnu": [],
"@rules_rust//rust/platform:x86_64-unknown-none": [],
"//conditions:default": ["@platforms//:incompatible"],
}),
version = "0.9.0",
deps = [
"@vendor__log-0.3.9//:log",
"@vendor__serde-1.0.216//:serde",
],
)

2
misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.ra_ap_load-cargo-0.0.248.bazel generated vendored
View File

@@ -91,7 +91,7 @@ rust_library(
}),
version = "0.0.248",
deps = [
"@vendor__anyhow-1.0.94//:anyhow",
"@vendor__anyhow-1.0.95//:anyhow",
"@vendor__crossbeam-channel-0.5.14//:crossbeam_channel",
"@vendor__itertools-0.12.1//:itertools",
"@vendor__ra_ap_hir_expand-0.0.248//:ra_ap_hir_expand",

2
misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.ra_ap_project_model-0.0.248.bazel generated vendored
View File

@@ -88,7 +88,7 @@ rust_library(
}),
version = "0.0.248",
deps = [
"@vendor__anyhow-1.0.94//:anyhow",
"@vendor__anyhow-1.0.95//:anyhow",
"@vendor__cargo_metadata-0.18.1//:cargo_metadata",
"@vendor__itertools-0.12.1//:itertools",
"@vendor__la-arena-0.3.1//:la_arena",

4
misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.serde_with-3.11.0.bazel → misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.serde_with-3.12.0.bazel generated vendored
View File

@@ -38,7 +38,7 @@ rust_library(
edition = "2021",
proc_macro_deps = [
"@vendor__serde_derive-1.0.216//:serde_derive",
"@vendor__serde_with_macros-3.11.0//:serde_with_macros",
"@vendor__serde_with_macros-3.12.0//:serde_with_macros",
],
rustc_flags = [
"--cap-lints=allow",
@@ -87,7 +87,7 @@ rust_library(
"@rules_rust//rust/platform:x86_64-unknown-none": [],
"//conditions:default": ["@platforms//:incompatible"],
}),
version = "3.11.0",
version = "3.12.0",
deps = [
"@vendor__serde-1.0.216//:serde",
],

2
misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.serde_with_macros-3.11.0.bazel → misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.serde_with_macros-3.12.0.bazel generated vendored
View File

@@ -77,7 +77,7 @@ rust_proc_macro(
"@rules_rust//rust/platform:x86_64-unknown-none": [],
"//conditions:default": ["@platforms//:incompatible"],
}),
version = "3.11.0",
version = "3.12.0",
deps = [
"@vendor__darling-0.20.10//:darling",
"@vendor__proc-macro2-1.0.92//:proc_macro2",

62
misc/bazel/3rdparty/tree_sitter_extractors_deps/defs.bzl generated vendored
View File

@@ -310,10 +310,13 @@ _NORMAL_DEPENDENCIES = {
},
"rust/ast-generator": {
_COMMON_CONDITION: {
"anyhow": Label("@vendor__anyhow-1.0.95//:anyhow"),
"either": Label("@vendor__either-1.13.0//:either"),
"itertools": Label("@vendor__itertools-0.12.1//:itertools"),
"mustache": Label("@vendor__mustache-0.9.0//:mustache"),
"proc-macro2": Label("@vendor__proc-macro2-1.0.92//:proc_macro2"),
"quote": Label("@vendor__quote-1.0.37//:quote"),
"serde": Label("@vendor__serde-1.0.216//:serde"),
"stdx": Label("@vendor__ra_ap_stdx-0.0.248//:ra_ap_stdx"),
"ungrammar": Label("@vendor__ungrammar-1.16.1//:ungrammar"),
},
@@ -322,7 +325,7 @@ _NORMAL_DEPENDENCIES = {
},
"rust/extractor": {
_COMMON_CONDITION: {
"anyhow": Label("@vendor__anyhow-1.0.94//:anyhow"),
"anyhow": Label("@vendor__anyhow-1.0.95//:anyhow"),
"argfile": Label("@vendor__argfile-0.2.1//:argfile"),
"chrono": Label("@vendor__chrono-0.4.39//:chrono"),
"clap": Label("@vendor__clap-4.5.23//:clap"),
@@ -348,7 +351,7 @@ _NORMAL_DEPENDENCIES = {
"ra_ap_vfs": Label("@vendor__ra_ap_vfs-0.0.248//:ra_ap_vfs"),
"serde": Label("@vendor__serde-1.0.216//:serde"),
"serde_json": Label("@vendor__serde_json-1.0.133//:serde_json"),
"serde_with": Label("@vendor__serde_with-3.11.0//:serde_with"),
"serde_with": Label("@vendor__serde_with-3.12.0//:serde_with"),
"stderrlog": Label("@vendor__stderrlog-0.6.0//:stderrlog"),
"triomphe": Label("@vendor__triomphe-0.1.14//:triomphe"),
},
@@ -739,12 +742,12 @@ def crate_repositories():
maybe(
http_archive,
name = "vendor__anyhow-1.0.94",
sha256 = "c1fd03a028ef38ba2276dce7e33fcd6369c158a1bca17946c4b1b701891c1ff7",
name = "vendor__anyhow-1.0.95",
sha256 = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04",
type = "tar.gz",
urls = ["https://static.crates.io/crates/anyhow/1.0.94/download"],
strip_prefix = "anyhow-1.0.94",
build_file = Label("//misc/bazel/3rdparty/tree_sitter_extractors_deps:BUILD.anyhow-1.0.94.bazel"),
urls = ["https://static.crates.io/crates/anyhow/1.0.95/download"],
strip_prefix = "anyhow-1.0.95",
build_file = Label("//misc/bazel/3rdparty/tree_sitter_extractors_deps:BUILD.anyhow-1.0.95.bazel"),
)
maybe(
@@ -1707,6 +1710,16 @@ def crate_repositories():
build_file = Label("//misc/bazel/3rdparty/tree_sitter_extractors_deps:BUILD.lock_api-0.4.12.bazel"),
)
maybe(
http_archive,
name = "vendor__log-0.3.9",
sha256 = "e19e8d5c34a3e0e2223db8e060f9e8264aeeb5c5fc64a4ee9965c062211c024b",
type = "tar.gz",
urls = ["https://static.crates.io/crates/log/0.3.9/download"],
strip_prefix = "log-0.3.9",
build_file = Label("//misc/bazel/3rdparty/tree_sitter_extractors_deps:BUILD.log-0.3.9.bazel"),
)
maybe(
http_archive,
name = "vendor__log-0.4.22",
@@ -1787,6 +1800,16 @@ def crate_repositories():
build_file = Label("//misc/bazel/3rdparty/tree_sitter_extractors_deps:BUILD.miow-0.6.0.bazel"),
)
maybe(
http_archive,
name = "vendor__mustache-0.9.0",
sha256 = "51956ef1c5d20a1384524d91e616fb44dfc7d8f249bf696d49c97dd3289ecab5",
type = "tar.gz",
urls = ["https://static.crates.io/crates/mustache/0.9.0/download"],
strip_prefix = "mustache-0.9.0",
build_file = Label("//misc/bazel/3rdparty/tree_sitter_extractors_deps:BUILD.mustache-0.9.0.bazel"),
)
maybe(
http_archive,
name = "vendor__nohash-hasher-0.2.0",
@@ -2568,22 +2591,22 @@ def crate_repositories():
maybe(
http_archive,
name = "vendor__serde_with-3.11.0",
sha256 = "8e28bdad6db2b8340e449f7108f020b3b092e8583a9e3fb82713e1d4e71fe817",
name = "vendor__serde_with-3.12.0",
sha256 = "d6b6f7f2fcb69f747921f79f3926bd1e203fce4fef62c268dd3abfb6d86029aa",
type = "tar.gz",
urls = ["https://static.crates.io/crates/serde_with/3.11.0/download"],
strip_prefix = "serde_with-3.11.0",
build_file = Label("//misc/bazel/3rdparty/tree_sitter_extractors_deps:BUILD.serde_with-3.11.0.bazel"),
urls = ["https://static.crates.io/crates/serde_with/3.12.0/download"],
strip_prefix = "serde_with-3.12.0",
build_file = Label("//misc/bazel/3rdparty/tree_sitter_extractors_deps:BUILD.serde_with-3.12.0.bazel"),
)
maybe(
http_archive,
name = "vendor__serde_with_macros-3.11.0",
sha256 = "9d846214a9854ef724f3da161b426242d8de7c1fc7de2f89bb1efcb154dca79d",
name = "vendor__serde_with_macros-3.12.0",
sha256 = "8d00caa5193a3c8362ac2b73be6b9e768aa5a4b2f721d8f4b339600c3cb51f8e",
type = "tar.gz",
urls = ["https://static.crates.io/crates/serde_with_macros/3.11.0/download"],
strip_prefix = "serde_with_macros-3.11.0",
build_file = Label("//misc/bazel/3rdparty/tree_sitter_extractors_deps:BUILD.serde_with_macros-3.11.0.bazel"),
urls = ["https://static.crates.io/crates/serde_with_macros/3.12.0/download"],
strip_prefix = "serde_with_macros-3.12.0",
build_file = Label("//misc/bazel/3rdparty/tree_sitter_extractors_deps:BUILD.serde_with_macros-3.12.0.bazel"),
)
maybe(
@@ -3357,7 +3380,7 @@ def crate_repositories():
)
return [
struct(repo = "vendor__anyhow-1.0.94", is_dev_dep = False),
struct(repo = "vendor__anyhow-1.0.95", is_dev_dep = False),
struct(repo = "vendor__argfile-0.2.1", is_dev_dep = False),
struct(repo = "vendor__chrono-0.4.39", is_dev_dep = False),
struct(repo = "vendor__clap-4.5.23", is_dev_dep = False),
@@ -3372,6 +3395,7 @@ def crate_repositories():
struct(repo = "vendor__itertools-0.13.0", is_dev_dep = False),
struct(repo = "vendor__lazy_static-1.5.0", is_dev_dep = False),
struct(repo = "vendor__log-0.4.22", is_dev_dep = False),
struct(repo = "vendor__mustache-0.9.0", is_dev_dep = False),
struct(repo = "vendor__num-traits-0.2.19", is_dev_dep = False),
struct(repo = "vendor__num_cpus-1.16.0", is_dev_dep = False),
struct(repo = "vendor__proc-macro2-1.0.92", is_dev_dep = False),
@@ -3395,7 +3419,7 @@ def crate_repositories():
struct(repo = "vendor__regex-1.11.1", is_dev_dep = False),
struct(repo = "vendor__serde-1.0.216", is_dev_dep = False),
struct(repo = "vendor__serde_json-1.0.133", is_dev_dep = False),
struct(repo = "vendor__serde_with-3.11.0", is_dev_dep = False),
struct(repo = "vendor__serde_with-3.12.0", is_dev_dep = False),
struct(repo = "vendor__stderrlog-0.6.0", is_dev_dep = False),
struct(repo = "vendor__syn-2.0.90", is_dev_dep = False),
struct(repo = "vendor__tracing-0.1.41", is_dev_dep = False),

1
rust/ast-generator/BUILD.bazel
View File

@@ -56,6 +56,7 @@ codeql_rust_binary(
) + [":codegen"],
aliases = aliases(),
args = ["$(rlocationpath :ungram)"],
compile_data = glob(["src/templates/*.mustache"]),
data = [":ungram"],
proc_macro_deps = all_crate_deps(
proc_macro = True,

3
rust/ast-generator/Cargo.toml
View File

@@ -11,3 +11,6 @@ quote = "1.0.20"
either = "1.9.0"
stdx = {package = "ra_ap_stdx", version = "0.0.248"}
itertools = "0.12.0"
mustache = "0.9.0"
serde = { version = "1.0.216", features = ["derive"] }
anyhow = "1.0.95"

587
rust/ast-generator/src/main.rs
View File

@@ -1,9 +1,11 @@
use std::io::Write;
use std::{fs, path::PathBuf};
pub mod codegen;
mod flags;
use crate::codegen::grammar::ast_src::{AstEnumSrc, Cardinality};
use codegen::grammar::ast_src::{AstNodeSrc, AstSrc, Field};
use itertools::Itertools;
use serde::Serialize;
use std::collections::{BTreeMap, BTreeSet};
use std::env;
use ungrammar::Grammar;
@@ -56,92 +58,116 @@ fn to_lower_snake_case(s: &str) -> String {
buf
}
#[derive(Serialize)]
struct SchemaField {
name: String,
ty: String,
child: bool,
}
#[derive(Serialize)]
struct SchemaClass {
name: String,
bases: Vec<String>,
fields: Vec<SchemaField>,
}
#[derive(Serialize, Default)]
struct Schema {
classes: Vec<SchemaClass>,
}
fn get_bases(name: &str, super_types: &BTreeMap<String, BTreeSet<String>>) -> Vec<String> {
super_types
.get(name)
.map(|tys| tys.iter().map(|t| class_name(t)).collect())
.unwrap_or_else(|| vec!["AstNode".to_string()])
}
fn enum_src_to_schema_class(
node: &AstEnumSrc,
super_types: &BTreeMap<String, BTreeSet<String>>,
) -> SchemaClass {
SchemaClass {
name: class_name(&node.name),
bases: get_bases(&node.name, super_types),
fields: Vec::new(),
}
}
fn node_src_to_schema_class(
node: &AstNodeSrc,
super_types: &BTreeMap<String, BTreeSet<String>>,
) -> SchemaClass {
SchemaClass {
name: class_name(&node.name),
bases: get_bases(&node.name, super_types),
fields: get_fields(node)
.iter()
.map(|f| {
let (ty, child) = match &f.ty {
FieldType::String => ("optional[string]".to_string(), false),
FieldType::Predicate => ("predicate".to_string(), false),
FieldType::Optional(ty) => (format!("optional[\"{}\"]", class_name(ty)), true),
FieldType::List(ty) => (format!("list[\"{}\"]", class_name(ty)), true),
};
SchemaField {
name: property_name(&node.name, &f.name),
ty,
child,
}
})
.collect(),
}
}
fn fix_blank_lines(s: &str) -> String {
// mustache is not very good at avoiding blank lines
// adopting the workaround from https://github.com/groue/GRMustache/issues/46#issuecomment-19498046
s.split("\n")
.filter(|line| !line.trim().is_empty())
.map(|line| if line == "" { "" } else { line })
.join("\n")
+ "\n"
}
fn write_schema(
grammar: &AstSrc,
super_types: BTreeMap<String, BTreeSet<String>>,
) -> std::io::Result<String> {
let mut buf: Vec<u8> = Vec::new();
writeln!(
buf,
"# Generated by `ast-generator`, do not edit by hand.\n"
)?;
writeln!(buf, "from .prelude import *")?;
) -> mustache::Result<String> {
let mut schema = Schema::default();
schema.classes.extend(
grammar
.enums
.iter()
.map(|node| enum_src_to_schema_class(node, &super_types)),
);
schema.classes.extend(
grammar
.nodes
.iter()
.map(|node| node_src_to_schema_class(node, &super_types)),
);
// the concat dance is currently required by bazel
let template = mustache::compile_str(include_str!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/src/templates/schema.mustache"
)))?;
let res = template.render_to_string(&schema)?;
Ok(fix_blank_lines(&res))
}
for node in &grammar.enums {
let super_classses = if let Some(cls) = super_types.get(&node.name) {
let super_classes: Vec<String> = cls.iter().map(|s| class_name(s)).collect();
super_classes.join(",")
} else {
"AstNode".to_owned()
};
writeln!(
buf,
"\nclass {}({}):",
class_name(&node.name),
super_classses
)?;
writeln!(buf, " pass")?;
}
for node in &grammar.nodes {
let super_classses = if let Some(cls) = super_types.get(&node.name) {
let super_classes: Vec<String> = cls.iter().map(|s| class_name(s)).collect();
super_classes.join(",")
} else {
"AstNode".to_owned()
};
writeln!(
buf,
"\nclass {}({}):",
class_name(&node.name),
super_classses
)?;
let mut empty = true;
for field in get_fields(node) {
if field.tp == "SyntaxToken" {
continue;
}
empty = false;
if field.tp == "predicate" {
writeln!(
buf,
" {}: predicate",
property_name(&node.name, &field.name),
)?;
} else if field.tp == "string" {
writeln!(
buf,
" {}: optional[string]",
property_name(&node.name, &field.name),
)?;
} else {
let list = field.is_many;
let (o, c) = if list {
("list[", "]")
} else {
("optional[", "]")
};
writeln!(
buf,
" {}: {}\"{}\"{} | child",
property_name(&node.name, &field.name),
o,
class_name(&field.tp),
c
)?;
};
}
if empty {
writeln!(buf, " pass")?;
}
}
Ok(String::from_utf8_lossy(&buf).to_string())
#[derive(Eq, PartialEq)]
enum FieldType {
String,
Predicate,
Optional(String),
List(String),
}
struct FieldInfo {
name: String,
tp: String,
is_many: bool,
ty: FieldType,
}
fn get_fields(node: &AstNodeSrc) -> Vec<FieldInfo> {
let mut result = Vec::new();
@@ -154,8 +180,7 @@ fn get_fields(node: &AstNodeSrc) -> Vec<FieldInfo> {
if predicates.contains(&name.as_str()) {
result.push(FieldInfo {
name: format!("is_{name}"),
tp: "predicate".to_string(),
is_many: false,
ty: FieldType::Predicate,
});
}
}
@@ -165,210 +190,177 @@ fn get_fields(node: &AstNodeSrc) -> Vec<FieldInfo> {
"Name" | "NameRef" | "Lifetime" => {
result.push(FieldInfo {
name: "text".to_string(),
tp: "string".to_string(),
is_many: false,
ty: FieldType::String,
});
}
"Abi" => {
result.push(FieldInfo {
name: "abi_string".to_string(),
tp: "string".to_string(),
is_many: false,
ty: FieldType::String,
});
}
"Literal" => {
result.push(FieldInfo {
name: "text_value".to_string(),
tp: "string".to_string(),
is_many: false,
ty: FieldType::String,
});
}
"PrefixExpr" => {
result.push(FieldInfo {
name: "operator_name".to_string(),
tp: "string".to_string(),
is_many: false,
ty: FieldType::String,
});
}
"BinExpr" => {
result.push(FieldInfo {
name: "lhs".to_string(),
tp: "Expr".to_string(),
is_many: false,
ty: FieldType::Optional("Expr".to_string()),
});
result.push(FieldInfo {
name: "rhs".to_string(),
tp: "Expr".to_string(),
is_many: false,
ty: FieldType::Optional("Expr".to_string()),
});
result.push(FieldInfo {
name: "operator_name".to_string(),
tp: "string".to_string(),
is_many: false,
ty: FieldType::String,
});
}
"IfExpr" => {
result.push(FieldInfo {
name: "then_branch".to_string(),
tp: "BlockExpr".to_string(),
is_many: false,
ty: FieldType::Optional("BlockExpr".to_string()),
});
result.push(FieldInfo {
name: "else_branch".to_string(),
tp: "ElseBranch".to_string(),
is_many: false,
ty: FieldType::Optional("ElseBranch".to_string()),
});
result.push(FieldInfo {
name: "condition".to_string(),
tp: "Expr".to_string(),
is_many: false,
ty: FieldType::Optional("Expr".to_string()),
});
}
"RangeExpr" => {
result.push(FieldInfo {
name: "start".to_string(),
tp: "Expr".to_string(),
is_many: false,
ty: FieldType::Optional("Expr".to_string()),
});
result.push(FieldInfo {
name: "end".to_string(),
tp: "Expr".to_string(),
is_many: false,
ty: FieldType::Optional("Expr".to_string()),
});
result.push(FieldInfo {
name: "operator_name".to_string(),
tp: "string".to_string(),
is_many: false,
ty: FieldType::String,
});
}
"RangePat" => {
result.push(FieldInfo {
name: "start".to_string(),
tp: "Pat".to_string(),
is_many: false,
ty: FieldType::Optional("Pat".to_string()),
});
result.push(FieldInfo {
name: "end".to_string(),
tp: "Pat".to_string(),
is_many: false,
ty: FieldType::Optional("Pat".to_string()),
});
result.push(FieldInfo {
name: "operator_name".to_string(),
tp: "string".to_string(),
is_many: false,
ty: FieldType::String,
});
}
"IndexExpr" => {
result.push(FieldInfo {
name: "index".to_string(),
tp: "Expr".to_string(),
is_many: false,
ty: FieldType::Optional("Expr".to_string()),
});
result.push(FieldInfo {
name: "base".to_string(),
tp: "Expr".to_string(),
is_many: false,
ty: FieldType::Optional("Expr".to_string()),
});
}
"Impl" => {
result.push(FieldInfo {
name: "trait_".to_string(),
tp: "Type".to_string(),
is_many: false,
ty: FieldType::Optional("Type".to_string()),
});
result.push(FieldInfo {
name: "self_ty".to_string(),
tp: "Type".to_string(),
is_many: false,
ty: FieldType::Optional("Type".to_string()),
});
}
"ForExpr" => {
result.push(FieldInfo {
name: "iterable".to_string(),
tp: "Expr".to_string(),
is_many: false,
ty: FieldType::Optional("Expr".to_string()),
});
}
"WhileExpr" => {
result.push(FieldInfo {
name: "condition".to_string(),
tp: "Expr".to_string(),
is_many: false,
ty: FieldType::Optional("Expr".to_string()),
});
}
"MatchGuard" => {
result.push(FieldInfo {
name: "condition".to_string(),
tp: "Expr".to_string(),
is_many: false,
ty: FieldType::Optional("Expr".to_string()),
});
}
"MacroDef" => {
result.push(FieldInfo {
name: "args".to_string(),
tp: "TokenTree".to_string(),
is_many: false,
ty: FieldType::Optional("TokenTree".to_string()),
});
result.push(FieldInfo {
name: "body".to_string(),
tp: "TokenTree".to_string(),
is_many: false,
ty: FieldType::Optional("TokenTree".to_string()),
});
}
"FormatArgsExpr" => {
result.push(FieldInfo {
name: "args".to_string(),
tp: "FormatArgsArg".to_string(),
is_many: true,
ty: FieldType::List("FormatArgsArg".to_string()),
});
}
"ArgList" => {
result.push(FieldInfo {
name: "args".to_string(),
tp: "Expr".to_string(),
is_many: true,
ty: FieldType::List("Expr".to_string()),
});
}
"Fn" => {
result.push(FieldInfo {
name: "body".to_string(),
tp: "BlockExpr".to_string(),
is_many: false,
ty: FieldType::Optional("BlockExpr".to_string()),
});
}
"Const" => {
result.push(FieldInfo {
name: "body".to_string(),
tp: "Expr".to_string(),
is_many: false,
ty: FieldType::Optional("Expr".to_string()),
});
}
"Static" => {
result.push(FieldInfo {
name: "body".to_string(),
tp: "Expr".to_string(),
is_many: false,
ty: FieldType::Optional("Expr".to_string()),
});
}
"ClosureExpr" => {
result.push(FieldInfo {
name: "body".to_string(),
tp: "Expr".to_string(),
is_many: false,
ty: FieldType::Optional("Expr".to_string()),
});
}
"ArrayExpr" => {
result.push(FieldInfo {
name: "is_semicolon".to_string(),
tp: "predicate".to_string(),
is_many: false,
ty: FieldType::Predicate,
});
}
"SelfParam" => {
result.push(FieldInfo {
name: "is_amp".to_string(),
tp: "predicate".to_string(),
is_many: false,
ty: FieldType::Predicate,
});
}
_ => {}
@@ -379,72 +371,69 @@ fn get_fields(node: &AstNodeSrc) -> Vec<FieldInfo> {
if node.name == "ArrayExpr" && field.method_name() == "expr" {
continue;
}
let ty = match field {
Field::Token(_) => continue,
Field::Node {
ty, cardinality, ..
} => match cardinality {
Cardinality::Optional => FieldType::Optional(ty.clone()),
Cardinality::Many => FieldType::List(ty.clone()),
},
};
result.push(FieldInfo {
name: field.method_name(),
tp: field.ty().to_string(),
is_many: field.is_many(),
ty,
});
}
for trait_ in &node.traits {
match trait_.as_str() {
"HasAttrs" => result.push(FieldInfo {
name: "attrs".to_owned(),
tp: "Attr".to_owned(),
is_many: true,
ty: FieldType::List("Attr".to_owned()),
}),
"HasName" => result.push(FieldInfo {
name: "name".to_owned(),
tp: "Name".to_owned(),
is_many: false,
ty: FieldType::Optional("Name".to_owned()),
}),
"HasVisibility" => result.push(FieldInfo {
name: "visibility".to_owned(),
tp: "Visibility".to_owned(),
is_many: false,
ty: FieldType::Optional("Visibility".to_owned()),
}),
"HasGenericParams" => {
result.push(FieldInfo {
name: "generic_param_list".to_owned(),
tp: "GenericParamList".to_owned(),
is_many: false,
ty: FieldType::Optional("GenericParamList".to_owned()),
});
result.push(FieldInfo {
name: "where_clause".to_owned(),
tp: "WhereClause".to_owned(),
is_many: false,
ty: FieldType::Optional("WhereClause".to_owned()),
})
}
"HasGenericArgs" => result.push(FieldInfo {
name: "generic_arg_list".to_owned(),
tp: "GenericArgList".to_owned(),
is_many: false,
ty: FieldType::Optional("GenericArgList".to_owned()),
}),
"HasTypeBounds" => result.push(FieldInfo {
name: "type_bound_list".to_owned(),
tp: "TypeBoundList".to_owned(),
is_many: false,
ty: FieldType::Optional("TypeBoundList".to_owned()),
}),
"HasModuleItem" => result.push(FieldInfo {
name: "items".to_owned(),
tp: "Item".to_owned(),
is_many: true,
ty: FieldType::List("Item".to_owned()),
}),
"HasLoopBody" => {
result.push(FieldInfo {
name: "label".to_owned(),
tp: "Label".to_owned(),
is_many: false,
ty: FieldType::Optional("Label".to_owned()),
});
result.push(FieldInfo {
name: "loop_body".to_owned(),
tp: "BlockExpr".to_owned(),
is_many: false,
ty: FieldType::Optional("BlockExpr".to_owned()),
})
}
"HasArgList" => result.push(FieldInfo {
name: "arg_list".to_owned(),
tp: "ArgList".to_owned(),
is_many: false,
ty: FieldType::Optional("ArgList".to_owned()),
}),
"HasDocComments" => {}
@@ -455,150 +444,122 @@ fn get_fields(node: &AstNodeSrc) -> Vec<FieldInfo> {
result
}
fn write_extractor(grammar: &AstSrc) -> std::io::Result<String> {
let mut buf: Vec<u8> = Vec::new();
writeln!(
buf,
"//! Generated by `ast-generator`, do not edit by hand.\n
#![cfg_attr(any(), rustfmt::skip)]
use super::base::Translator;
use super::mappings::TextValue;
use crate::emit_detached;
use crate::generated;
use crate::trap::{{Label, TrapId}};
use ra_ap_syntax::ast::{{
HasArgList, HasAttrs, HasGenericArgs, HasGenericParams, HasLoopBody, HasModuleItem, HasName,
HasTypeBounds, HasVisibility, RangeItem,
}};
use ra_ap_syntax::{{ast, AstNode}};
impl Translator<'_> {{
fn emit_else_branch(&mut self, node: ast::ElseBranch) -> Option<Label<generated::Expr>> {{
match node {{
ast::ElseBranch::IfExpr(inner) => self.emit_if_expr(inner).map(Into::into),
ast::ElseBranch::Block(inner) => self.emit_block_expr(inner).map(Into::into),
}}
}}\n"
)?;
for node in &grammar.enums {
let type_name = &node.name;
let class_name = class_name(&node.name);
writeln!(
buf,
" pub(crate) fn emit_{}(&mut self, node: ast::{}) -> Option<Label<generated::{}>> {{",
to_lower_snake_case(type_name),
type_name,
class_name
)?;
writeln!(buf, " match node {{")?;
for variant in &node.variants {
writeln!(
buf,
" ast::{}::{}(inner) => self.emit_{}(inner).map(Into::into),",
type_name,
variant,
to_lower_snake_case(variant)
)?;
}
writeln!(buf, " }}")?;
writeln!(buf, " }}\n")?;
}
for node in &grammar.nodes {
let type_name = &node.name;
let class_name = class_name(&node.name);
writeln!(
buf,
" pub(crate) fn emit_{}(&mut self, node: ast::{}) -> Option<Label<generated::{}>> {{",
to_lower_snake_case(type_name),
type_name,
class_name
)?;
for field in get_fields(node) {
if &field.tp == "SyntaxToken" {
continue;
}
if field.name == "attrs" {
// special case: this means the node type implements `HasAttrs`, and we want to
// check whether it was not excluded by a `cfg` attribute
writeln!(
buf,
" if self.should_be_excluded(&node) {{ return None; }}"
)?;
}
let type_name = &field.tp;
let struct_field_name = &field.name;
let class_field_name = property_name(&node.name, &field.name);
if field.tp == "predicate" {
writeln!(
buf,
" let {} = node.{}_token().is_some();",
class_field_name,
&struct_field_name[3..],
)?;
} else if field.tp == "string" {
writeln!(
buf,
" let {} = node.try_get_text();",
class_field_name,
)?;
} else if field.is_many {
writeln!(
buf,
" let {} = node.{}().filter_map(|x| self.emit_{}(x)).collect();",
class_field_name,
struct_field_name,
to_lower_snake_case(type_name)
)?;
} else {
writeln!(
buf,
" let {} = node.{}().and_then(|x| self.emit_{}(x));",
class_field_name,
struct_field_name,
to_lower_snake_case(type_name)
)?;
}
}
writeln!(
buf,
" let label = self.trap.emit(generated::{} {{",
class_name
)?;
writeln!(buf, " id: TrapId::Star,")?;
for field in get_fields(node) {
if field.tp == "SyntaxToken" {
continue;
}
let class_field_name: String = property_name(&node.name, &field.name);
writeln!(buf, " {},", class_field_name)?;
}
writeln!(buf, " }});")?;
writeln!(buf, " self.emit_location(label, &node);")?;
writeln!(
buf,
" emit_detached!({}, self, node, label);",
class_name
)?;
writeln!(
buf,
" self.emit_tokens(&node, label.into(), node.syntax().children_with_tokens());"
)?;
writeln!(buf, " Some(label)")?;
writeln!(buf, " }}\n")?;
}
writeln!(buf, "}}")?;
Ok(String::from_utf8_lossy(&buf).into_owned())
#[derive(Serialize)]
struct EnumVariantInfo {
name: String,
snake_case_name: String,
}
fn main() -> std::io::Result<()> {
#[derive(Serialize)]
struct ExtractorEnumInfo {
name: String,
snake_case_name: String,
ast_name: String,
variants: Vec<EnumVariantInfo>,
}
#[derive(Serialize, Default)]
struct ExtractorNodeFieldInfo {
name: String,
method: String,
snake_case_ty: String,
string: bool,
predicate: bool,
optional: bool,
list: bool,
}
#[derive(Serialize)]
struct ExtractorNodeInfo {
name: String,
snake_case_name: String,
ast_name: String,
fields: Vec<ExtractorNodeFieldInfo>,
has_attrs: bool,
}
#[derive(Serialize)]
struct ExtractorInfo {
enums: Vec<ExtractorEnumInfo>,
nodes: Vec<ExtractorNodeInfo>,
}
fn enum_to_extractor_info(node: &AstEnumSrc) -> ExtractorEnumInfo {
ExtractorEnumInfo {
name: class_name(&node.name),
snake_case_name: to_lower_snake_case(&node.name),
ast_name: node.name.clone(),
variants: node
.variants
.iter()
.map(|v| EnumVariantInfo {
name: v.clone(),
snake_case_name: to_lower_snake_case(v),
})
.collect(),
}
}
fn field_info_to_extractor_info(node: &AstNodeSrc, field: &FieldInfo) -> ExtractorNodeFieldInfo {
let name = property_name(&node.name, &field.name);
match &field.ty {
FieldType::String => ExtractorNodeFieldInfo {
name,
string: true,
..Default::default()
},
FieldType::Predicate => ExtractorNodeFieldInfo {
name,
method: format!("{}_token", &field.name[3..]),
predicate: true,
..Default::default()
},
FieldType::Optional(ty) => ExtractorNodeFieldInfo {
name,
method: field.name.clone(),
snake_case_ty: to_lower_snake_case(ty),
optional: true,
..Default::default()
},
FieldType::List(ty) => ExtractorNodeFieldInfo {
name,
method: field.name.clone(),
snake_case_ty: to_lower_snake_case(ty),
list: true,
..Default::default()
},
}
}
fn node_to_extractor_info(node: &AstNodeSrc) -> ExtractorNodeInfo {
let fields = get_fields(node);
let has_attrs = fields.iter().any(|f| f.name == "attrs");
ExtractorNodeInfo {
name: class_name(&node.name),
snake_case_name: to_lower_snake_case(&node.name),
ast_name: node.name.clone(),
fields: fields
.iter()
.map(|f| field_info_to_extractor_info(node, f))
.collect(),
has_attrs,
}
}
fn write_extractor(grammar: &AstSrc) -> mustache::Result<String> {
let extractor_info = ExtractorInfo {
enums: grammar.enums.iter().map(enum_to_extractor_info).collect(),
nodes: grammar.nodes.iter().map(node_to_extractor_info).collect(),
};
// the concat dance is currently required by bazel
let template = mustache::compile_str(include_str!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/src/templates/extractor.mustache"
)))?;
let res = template.render_to_string(&extractor_info)?;
Ok(fix_blank_lines(&res))
}
fn main() -> anyhow::Result<()> {
let grammar = PathBuf::from("..").join(env::args().nth(1).expect("grammar file path required"));
let grammar: Grammar = fs::read_to_string(&grammar)
.unwrap_or_else(|_| panic!("Failed to parse grammar file: {}", grammar.display()))

65
rust/ast-generator/src/templates/extractor.mustache Normal file
View File

@@ -0,0 +1,65 @@
//! Generated by `ast-generator`, do not edit by hand.
¶{{! <- denotes empty line that should be kept, all blank lines are removed otherwise}}
#![cfg_attr(any(), rustfmt::skip)]
use super::base::Translator;
use super::mappings::TextValue;
use crate::emit_detached;
use crate::generated;
use crate::trap::{Label, TrapId};
use ra_ap_syntax::ast::{
HasArgList, HasAttrs, HasGenericArgs, HasGenericParams, HasLoopBody, HasModuleItem, HasName,
HasTypeBounds, HasVisibility, RangeItem,
};
use ra_ap_syntax::{ast, AstNode};
impl Translator<'_> {
fn emit_else_branch(&mut self, node: ast::ElseBranch) -> Option<Label<generated::Expr>> {
match node {
ast::ElseBranch::IfExpr(inner) => self.emit_if_expr(inner).map(Into::into),
ast::ElseBranch::Block(inner) => self.emit_block_expr(inner).map(Into::into),
}
}
{{#enums}}
pub(crate) fn emit_{{snake_case_name}}(&mut self, node: ast::{{ast_name}}) -> Option<Label<generated::{{name}}>> {
match node {
{{#variants}}
ast::{{ast_name}}::{{name}}(inner) => self.emit_{{snake_case_name}}(inner).map(Into::into),
{{/variants}}
}
}
{{/enums}}
{{#nodes}}
pub(crate) fn emit_{{snake_case_name}}(&mut self, node: ast::{{ast_name}}) -> Option<Label<generated::{{name}}>> {
{{#has_attrs}}
if self.should_be_excluded(&node) { return None; }
{{/has_attrs}}
{{#fields}}
{{#predicate}}
let {{name}} = node.{{method}}().is_some();
{{/predicate}}
{{#string}}
let {{name}} = node.try_get_text();
{{/string}}
{{#list}}
let {{name}} = node.{{method}}().filter_map(|x| self.emit_{{snake_case_ty}}(x)).collect();
{{/list}}
{{#optional}}
let {{name}} = node.{{method}}().and_then(|x| self.emit_{{snake_case_ty}}(x));
{{/optional}}
{{/fields}}
let label = self.trap.emit(generated::{{name}} {
id: TrapId::Star,
{{#fields}}
{{name}},
{{/fields}}
});
self.emit_location(label, &node);
emit_detached!({{name}}, self, node, label);
self.emit_tokens(&node, label.into(), node.syntax().children_with_tokens());
Some(label)
}
{{/nodes}}
}

13
rust/ast-generator/src/templates/schema.mustache Normal file
View File

@@ -0,0 +1,13 @@
# Generated by `ast-generator`, do not edit by hand.
¶{{! <- denotes empty line that should be kept, all blank lines are removed otherwise}}
from .prelude import *
{{#classes}}
class {{name}}({{#bases}}{{.}}, {{/bases}}):
{{#fields}}
{{name}}: {{{ty}}}{{#child}} | child{{/child}}
{{/fields}}
{{^fields}}
pass
{{/fields}}
{{/classes}}

2
rust/extractor/Cargo.toml
View File

@@ -5,7 +5,7 @@ edition = "2021"
# When updating these dependencies, run `rust/update_cargo_deps.sh`
[dependencies]
anyhow = "1.0.86"
anyhow = "1.0.95"
clap = { version = "4.5.16", features = ["derive"] }
figment = { version = "0.10.19", features = ["env", "yaml"] }
log = "0.4.22"

23
rust/extractor/src/translate/generated.rs generated
View File

@@ -288,8 +288,8 @@ impl Translator<'_> {
}
pub(crate) fn emit_asm_expr(&mut self, node: ast::AsmExpr) -> Option<Label<generated::AsmExpr>> {
let asm_pieces = node.asm_pieces().filter_map(|x| self.emit_asm_piece(x)).collect();
if self.should_be_excluded(&node) { return None; }
let asm_pieces = node.asm_pieces().filter_map(|x| self.emit_asm_piece(x)).collect();
let attrs = node.attrs().filter_map(|x| self.emit_attr(x)).collect();
let template = node.template().filter_map(|x| self.emit_expr(x)).collect();
let label = self.trap.emit(generated::AsmExpr {
@@ -409,8 +409,8 @@ impl Translator<'_> {
}
pub(crate) fn emit_assoc_item_list(&mut self, node: ast::AssocItemList) -> Option<Label<generated::AssocItemList>> {
let assoc_items = node.assoc_items().filter_map(|x| self.emit_assoc_item(x)).collect();
if self.should_be_excluded(&node) { return None; }
let assoc_items = node.assoc_items().filter_map(|x| self.emit_assoc_item(x)).collect();
let attrs = node.attrs().filter_map(|x| self.emit_attr(x)).collect();
let label = self.trap.emit(generated::AssocItemList {
id: TrapId::Star,
@@ -569,8 +569,8 @@ impl Translator<'_> {
}
pub(crate) fn emit_call_expr(&mut self, node: ast::CallExpr) -> Option<Label<generated::CallExpr>> {
let arg_list = node.arg_list().and_then(|x| self.emit_arg_list(x));
if self.should_be_excluded(&node) { return None; }
let arg_list = node.arg_list().and_then(|x| self.emit_arg_list(x));
let attrs = node.attrs().filter_map(|x| self.emit_attr(x)).collect();
let function = node.expr().and_then(|x| self.emit_expr(x));
let label = self.trap.emit(generated::CallExpr {
@@ -780,8 +780,8 @@ impl Translator<'_> {
}
pub(crate) fn emit_extern_block(&mut self, node: ast::ExternBlock) -> Option<Label<generated::ExternBlock>> {
let abi = node.abi().and_then(|x| self.emit_abi(x));
if self.should_be_excluded(&node) { return None; }
let abi = node.abi().and_then(|x| self.emit_abi(x));
let attrs = node.attrs().filter_map(|x| self.emit_attr(x)).collect();
let extern_item_list = node.extern_item_list().and_then(|x| self.emit_extern_item_list(x));
let is_unsafe = node.unsafe_token().is_some();
@@ -850,8 +850,8 @@ impl Translator<'_> {
}
pub(crate) fn emit_fn(&mut self, node: ast::Fn) -> Option<Label<generated::Function>> {
let abi = node.abi().and_then(|x| self.emit_abi(x));
if self.should_be_excluded(&node) { return None; }
let abi = node.abi().and_then(|x| self.emit_abi(x));
let attrs = node.attrs().filter_map(|x| self.emit_attr(x)).collect();
let body = node.body().and_then(|x| self.emit_block_expr(x));
let generic_param_list = node.generic_param_list().and_then(|x| self.emit_generic_param_list(x));
@@ -960,8 +960,8 @@ impl Translator<'_> {
}
pub(crate) fn emit_format_args_expr(&mut self, node: ast::FormatArgsExpr) -> Option<Label<generated::FormatArgsExpr>> {
let args = node.args().filter_map(|x| self.emit_format_args_arg(x)).collect();
if self.should_be_excluded(&node) { return None; }
let args = node.args().filter_map(|x| self.emit_format_args_arg(x)).collect();
let attrs = node.attrs().filter_map(|x| self.emit_attr(x)).collect();
let template = node.template().and_then(|x| self.emit_expr(x));
let label = self.trap.emit(generated::FormatArgsExpr {
@@ -1041,8 +1041,8 @@ impl Translator<'_> {
}
pub(crate) fn emit_impl(&mut self, node: ast::Impl) -> Option<Label<generated::Impl>> {
let assoc_item_list = node.assoc_item_list().and_then(|x| self.emit_assoc_item_list(x));
if self.should_be_excluded(&node) { return None; }
let assoc_item_list = node.assoc_item_list().and_then(|x| self.emit_assoc_item_list(x));
let attrs = node.attrs().filter_map(|x| self.emit_attr(x)).collect();
let generic_param_list = node.generic_param_list().and_then(|x| self.emit_generic_param_list(x));
let is_const = node.const_token().is_some();
@@ -1290,8 +1290,8 @@ impl Translator<'_> {
}
pub(crate) fn emit_macro_def(&mut self, node: ast::MacroDef) -> Option<Label<generated::MacroDef>> {
let args = node.args().and_then(|x| self.emit_token_tree(x));
if self.should_be_excluded(&node) { return None; }
let args = node.args().and_then(|x| self.emit_token_tree(x));
let attrs = node.attrs().filter_map(|x| self.emit_attr(x)).collect();
let body = node.body().and_then(|x| self.emit_token_tree(x));
let name = node.name().and_then(|x| self.emit_name(x));
@@ -1411,8 +1411,8 @@ impl Translator<'_> {
}
pub(crate) fn emit_match_arm_list(&mut self, node: ast::MatchArmList) -> Option<Label<generated::MatchArmList>> {
let arms = node.arms().filter_map(|x| self.emit_match_arm(x)).collect();
if self.should_be_excluded(&node) { return None; }
let arms = node.arms().filter_map(|x| self.emit_match_arm(x)).collect();
let attrs = node.attrs().filter_map(|x| self.emit_attr(x)).collect();
let label = self.trap.emit(generated::MatchArmList {
id: TrapId::Star,
@@ -1473,8 +1473,8 @@ impl Translator<'_> {
}
pub(crate) fn emit_method_call_expr(&mut self, node: ast::MethodCallExpr) -> Option<Label<generated::MethodCallExpr>> {
let arg_list = node.arg_list().and_then(|x| self.emit_arg_list(x));
if self.should_be_excluded(&node) { return None; }
let arg_list = node.arg_list().and_then(|x| self.emit_arg_list(x));
let attrs = node.attrs().filter_map(|x| self.emit_attr(x)).collect();
let generic_arg_list = node.generic_arg_list().and_then(|x| self.emit_generic_arg_list(x));
let name_ref = node.name_ref().and_then(|x| self.emit_name_ref(x));
@@ -2179,8 +2179,8 @@ impl Translator<'_> {
}
pub(crate) fn emit_trait(&mut self, node: ast::Trait) -> Option<Label<generated::Trait>> {
let assoc_item_list = node.assoc_item_list().and_then(|x| self.emit_assoc_item_list(x));
if self.should_be_excluded(&node) { return None; }
let assoc_item_list = node.assoc_item_list().and_then(|x| self.emit_assoc_item_list(x));
let attrs = node.attrs().filter_map(|x| self.emit_attr(x)).collect();
let generic_param_list = node.generic_param_list().and_then(|x| self.emit_generic_param_list(x));
let is_auto = node.auto_token().is_some();
@@ -2643,5 +2643,4 @@ impl Translator<'_> {
self.emit_tokens(&node, label.into(), node.syntax().children_with_tokens());
Some(label)
}
}

1500
rust/schema/ast.py generated
View File

File diff suppressed because it is too large Load Diff