hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2026-07-10 05:55:32 +02:00
Code Issues Packages Projects Releases Wiki Activity

yeast: Move schema and YAML loader into yeast-schema crate

Browse Source 
For type checking rules, we need to be able to load schemas (so we know
what to check against). However, since we can't have yeast-macros
depending on yeast (where the schema-handling code currently lives) as
this would introduce a circular dependency, we instead split the
schema-related code into its own yeast-schema crate.
This commit is contained in:
Taus
2026-06-19 14:53:35 +00:00
parent 4f4cdf434b
commit e6b865335c
13 changed files with 1268 additions and 1040 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
Cargo.lock generated
View File

@@ -3724,6 +3724,7 @@ dependencies = [
"tree-sitter-python",
"tree-sitter-ruby",
"yeast-macros",
"yeast-schema",
]
[[package]]
@@ -3735,6 +3736,15 @@ dependencies = [
"syn",
]
[[package]]
name = "yeast-schema"
version = "0.1.0"
dependencies = [
"serde",
"serde_json",
"serde_yaml",
]
[[package]]
name = "yoke"
version = "0.8.0"

1
Cargo.toml
View File

@@ -6,6 +6,7 @@ members = [
"shared/tree-sitter-extractor",
"shared/yeast",
"shared/yeast-macros",
"shared/yeast-schema",
"ruby/extractor",
"unified/extractor",
"unified/extractor/tree-sitter-swift",

31
misc/bazel/3rdparty/tree_sitter_extractors_deps/defs.bzl generated vendored
View File

@@ -403,6 +403,13 @@ _NORMAL_DEPENDENCIES = {
"syn": Label("@vendor_ts__syn-2.0.106//:syn"),
},
},
"shared/yeast-schema": {
_COMMON_CONDITION: {
"serde": Label("@vendor_ts__serde-1.0.228//:serde"),
"serde_json": Label("@vendor_ts__serde_json-1.0.145//:serde_json"),
"serde_yaml": Label("@vendor_ts__serde_yaml-0.9.34-deprecated//:serde_yaml"),
},
},
"unified/extractor": {
_COMMON_CONDITION: {
"clap": Label("@vendor_ts__clap-4.5.48//:clap"),
@@ -456,6 +463,10 @@ _NORMAL_ALIASES = {
_COMMON_CONDITION: {
},
},
"shared/yeast-schema": {
_COMMON_CONDITION: {
},
},
"unified/extractor": {
_COMMON_CONDITION: {
},
@@ -488,6 +499,8 @@ _NORMAL_DEV_DEPENDENCIES = {
},
"shared/yeast-macros": {
},
"shared/yeast-schema": {
},
"unified/extractor": {
},
"unified/extractor/tree-sitter-swift": {
@@ -513,6 +526,8 @@ _NORMAL_DEV_ALIASES = {
},
"shared/yeast-macros": {
},
"shared/yeast-schema": {
},
"unified/extractor": {
},
"unified/extractor/tree-sitter-swift": {
@@ -536,6 +551,8 @@ _PROC_MACRO_DEPENDENCIES = {
},
"shared/yeast-macros": {
},
"shared/yeast-schema": {
},
"unified/extractor": {
},
"unified/extractor/tree-sitter-swift": {
@@ -559,6 +576,8 @@ _PROC_MACRO_ALIASES = {
},
"shared/yeast-macros": {
},
"shared/yeast-schema": {
},
"unified/extractor": {
},
"unified/extractor/tree-sitter-swift": {
@@ -582,6 +601,8 @@ _PROC_MACRO_DEV_DEPENDENCIES = {
},
"shared/yeast-macros": {
},
"shared/yeast-schema": {
},
"unified/extractor": {
},
"unified/extractor/tree-sitter-swift": {
@@ -607,6 +628,8 @@ _PROC_MACRO_DEV_ALIASES = {
},
"shared/yeast-macros": {
},
"shared/yeast-schema": {
},
"unified/extractor": {
},
"unified/extractor/tree-sitter-swift": {
@@ -630,6 +653,8 @@ _BUILD_DEPENDENCIES = {
},
"shared/yeast-macros": {
},
"shared/yeast-schema": {
},
"unified/extractor": {
},
"unified/extractor/tree-sitter-swift": {
@@ -657,6 +682,8 @@ _BUILD_ALIASES = {
},
"shared/yeast-macros": {
},
"shared/yeast-schema": {
},
"unified/extractor": {
},
"unified/extractor/tree-sitter-swift": {
@@ -682,6 +709,8 @@ _BUILD_PROC_MACRO_DEPENDENCIES = {
},
"shared/yeast-macros": {
},
"shared/yeast-schema": {
},
"unified/extractor": {
},
"unified/extractor/tree-sitter-swift": {
@@ -705,6 +734,8 @@ _BUILD_PROC_MACRO_ALIASES = {
},
"shared/yeast-macros": {
},
"shared/yeast-schema": {
},
"unified/extractor": {
},
"unified/extractor/tree-sitter-swift": {

12
shared/yeast-schema/BUILD.bazel Normal file
View File

@@ -0,0 +1,12 @@
load("@rules_rust//rust:defs.bzl", "rust_library")
load("//misc/bazel/3rdparty/tree_sitter_extractors_deps:defs.bzl", "aliases", "all_crate_deps")
exports_files(["Cargo.toml"])
rust_library(
name = "yeast-schema",
srcs = glob(["src/**/*.rs"]),
aliases = aliases(),
visibility = ["//visibility:public"],
deps = all_crate_deps(),
)

9
shared/yeast-schema/Cargo.toml Normal file
View File

@@ -0,0 +1,9 @@
[package]
name = "yeast-schema"
version = "0.1.0"
edition = "2021"
[dependencies]
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
serde_yaml = "0.9"

33
shared/yeast-schema/src/lib.rs Normal file
View File

@@ -0,0 +1,33 @@
//! Schema definitions and YAML/JSON node-types loaders for YEAST.
//!
//! This crate carries the parts of the YEAST framework that don't need
//! `tree-sitter`: the [`schema::Schema`] type and its associated
//! [`schema::NodeType`] / [`schema::FieldCardinality`] helpers, plus the
//! YAML and JSON conversion helpers in [`node_types_yaml`].
//!
//! It exists so that both the runtime crate (`yeast`) and the
//! compile-time `rules!` proc macro (`yeast-macros`) can build against a
//! single source of truth without dragging tree-sitter (a heavy C-backed
//! dep) into the proc-macro toolchain.
//!
//! Tree-sitter-aware adapters — building a `Schema` from a
//! `tree_sitter::Language`, or loading a YAML schema on top of one —
//! live in `yeast::schema` and `yeast::node_types_yaml` respectively.
pub mod node_types_yaml;
pub mod schema;
/// Field IDs are stable `u16`s, matching tree-sitter's representation so a
/// schema built from a tree-sitter language can preserve the language's
/// existing IDs.
pub type FieldId = u16;
/// Kind IDs are stable `u16`s. Like `FieldId`, this matches tree-sitter's
/// representation.
pub type KindId = u16;
/// Sentinel field id used to mean "the implicit unfielded slot" (what the
/// tree-sitter docs call `children` and what YEAST surfaces in queries as
/// the bare `child:` field). Reserved to avoid clashing with real field
/// IDs allocated by `Schema::register_field`.
pub const CHILD_FIELD: u16 = u16::MAX;

762
shared/yeast-schema/src/node_types_yaml.rs Normal file
View File

@@ -0,0 +1,762 @@
/// Converts a YAML node-types file to the tree-sitter `node-types.json` format.
///
/// # YAML format
///
/// ```yaml
/// supertypes:
/// _expression:
/// - assignment
/// - binary
///
/// named:
/// assignment:
/// left: _lhs
/// right: _expression
/// identifier:
///
/// unnamed:
/// - "+"
/// - "end"
/// ```
///
/// See the crate-level docs for the full format specification.
use std::collections::{BTreeMap, BTreeSet};
use std::fmt::Write;
use crate::CHILD_FIELD;
use serde::Deserialize;
use serde_json::json;
/// Top-level YAML structure.
#[derive(Deserialize, Default)]
struct YamlNodeTypes {
#[serde(default)]
supertypes: BTreeMap<String, Vec<TypeRef>>,
#[serde(default)]
named: BTreeMap<String, Option<BTreeMap<String, TypeRefOrList>>>,
#[serde(default)]
unnamed: Vec<String>,
}
/// A reference to a node type. Can be:
/// - a plain string (resolved by looking up named vs unnamed)
/// - a map `{unnamed: "name"}` to force unnamed interpretation
#[derive(Deserialize, Debug, Clone)]
#[serde(untagged)]
enum TypeRef {
Name(String),
Explicit { unnamed: String },
}
/// A field value: either a single type ref or a list of them.
#[derive(Deserialize, Debug, Clone)]
#[serde(untagged)]
enum TypeRefOrList {
Single(TypeRef),
List(Vec<TypeRef>),
}
impl TypeRefOrList {
fn into_vec(self) -> Vec<TypeRef> {
match self {
TypeRefOrList::Single(t) => vec![t],
TypeRefOrList::List(v) => v,
}
}
}
/// Parsed field name: base name + multiplicity markers.
struct FieldSpec {
name: Option<String>, // None for $children
multiple: bool,
required: bool,
}
fn parse_field_name(raw: &str) -> FieldSpec {
let is_children =
raw == "$children" || raw == "$children?" || raw == "$children*" || raw == "$children+";
let suffix = raw.chars().last().filter(|c| matches!(c, '?' | '*' | '+'));
let (multiple, required) = match suffix {
Some('?') => (false, false),
Some('*') => (true, false),
Some('+') => (true, true),
_ => (false, true), // bare field name = required, single
};
let name = if is_children {
None
} else {
let base = raw.trim_end_matches(['?', '*', '+']);
Some(base.to_string())
};
FieldSpec {
name,
multiple,
required,
}
}
/// Resolve a TypeRef to a (type, named) pair, given the sets of known named
/// and unnamed types.
fn resolve_type_ref_pair(
type_ref: &TypeRef,
named_types: &BTreeSet<String>,
unnamed_types: &BTreeSet<String>,
) -> (String, bool) {
match type_ref {
TypeRef::Explicit { unnamed } => (unnamed.clone(), false),
TypeRef::Name(name) => {
let is_named = named_types.contains(name);
let is_unnamed = unnamed_types.contains(name);
if is_named && is_unnamed {
(name.clone(), true)
} else if is_unnamed {
(name.clone(), false)
} else {
(name.clone(), true)
}
}
}
}
/// Resolve a TypeRef to a {type, named} JSON record, given the sets of known named
/// and unnamed types.
fn resolve_type_ref(
type_ref: &TypeRef,
named_types: &BTreeSet<String>,
unnamed_types: &BTreeSet<String>,
) -> serde_json::Value {
let (kind, named) = resolve_type_ref_pair(type_ref, named_types, unnamed_types);
json!({"type": kind, "named": named})
}
/// Convert YAML string to node-types JSON string.
pub fn convert(yaml_input: &str) -> Result<String, String> {
let yaml: YamlNodeTypes =
serde_yaml::from_str(yaml_input).map_err(|e| format!("Failed to parse YAML: {e}"))?;
// Build the sets of known named and unnamed types for resolution.
let mut named_types = BTreeSet::new();
for name in yaml.supertypes.keys() {
named_types.insert(name.clone());
}
for name in yaml.named.keys() {
named_types.insert(name.clone());
}
let unnamed_types: BTreeSet<String> = yaml.unnamed.iter().cloned().collect();
let mut output = Vec::new();
// 1. Supertypes
for (name, members) in &yaml.supertypes {
let subtypes: Vec<_> = members
.iter()
.map(|m| resolve_type_ref(m, &named_types, &unnamed_types))
.collect();
output.push(json!({
"type": name,
"named": true,
"subtypes": subtypes,
}));
}
// 2. Named nodes
for (name, fields_opt) in &yaml.named {
let fields_map = match fields_opt {
None => {
// Leaf token: no fields, no children, no subtypes
output.push(json!({
"type": name,
"named": true,
"fields": {},
}));
continue;
}
Some(m) if m.is_empty() => {
output.push(json!({
"type": name,
"named": true,
"fields": {},
}));
continue;
}
Some(m) => m,
};
let mut json_fields = serde_json::Map::new();
let mut json_children: Option<serde_json::Value> = None;
for (raw_field_name, type_refs) in fields_map {
let spec = parse_field_name(raw_field_name);
let types: Vec<_> = type_refs
.clone()
.into_vec()
.iter()
.map(|t| resolve_type_ref(t, &named_types, &unnamed_types))
.collect();
// Cloning to make the borrow checker happy
let field_info = json!({
"multiple": spec.multiple,
"required": spec.required,
"types": types,
});
if spec.name.is_none() {
// $children
json_children = Some(field_info);
} else {
json_fields.insert(spec.name.unwrap(), field_info);
}
}
let mut entry = json!({
"type": name,
"named": true,
"fields": json_fields,
});
if let Some(children) = json_children {
entry
.as_object_mut()
.unwrap()
.insert("children".to_string(), children);
}
output.push(entry);
}
// 3. Unnamed tokens
for name in &yaml.unnamed {
output.push(json!({
"type": name,
"named": false,
}));
}
serde_json::to_string_pretty(&output).map_err(|e| format!("Failed to serialize JSON: {e}"))
}
/// Apply YAML node-type definitions to a mutable Schema.
/// Registers all types, fields, and allowed types from the YAML into the
/// schema. Public so callers can layer YAML node-types onto a Schema that
/// already has fields/kinds preregistered from another source (e.g. a
/// tree-sitter language).
pub fn extend_schema_from_yaml(
schema: &mut crate::schema::Schema,
yaml_input: &str,
) -> Result<(), String> {
let yaml: YamlNodeTypes =
serde_yaml::from_str(yaml_input).map_err(|e| format!("Failed to parse YAML: {e}"))?;
apply_yaml_to_schema(&yaml, schema);
Ok(())
}
fn apply_yaml_to_schema(
yaml: &YamlNodeTypes,
schema: &mut crate::schema::Schema,
) {
// Register all supertypes as node kinds
for name in yaml.supertypes.keys() {
schema.register_kind(name);
}
// Register named node kinds and their fields
for (name, fields_opt) in &yaml.named {
schema.register_kind(name);
if let Some(fields) = fields_opt {
for raw_field_name in fields.keys() {
let spec = parse_field_name(raw_field_name);
if let Some(field_name) = &spec.name {
schema.register_field(field_name);
}
}
}
}
// Register unnamed tokens as node kinds
for name in &yaml.unnamed {
schema.register_unnamed_kind(name);
}
let mut named_types = BTreeSet::new();
for name in yaml.supertypes.keys() {
named_types.insert(name.clone());
}
for name in yaml.named.keys() {
named_types.insert(name.clone());
}
let unnamed_types: BTreeSet<String> = yaml.unnamed.iter().cloned().collect();
for (supertype, members) in &yaml.supertypes {
let node_types = members
.iter()
.map(|m| {
let (kind, named) = resolve_type_ref_pair(m, &named_types, &unnamed_types);
crate::schema::NodeType { kind, named }
})
.collect();
schema.set_supertype_members(supertype, node_types);
}
// Register allowed field child types for type checking.
for (parent_kind, fields_opt) in &yaml.named {
let Some(fields) = fields_opt else {
continue;
};
for (raw_field_name, type_refs) in fields {
let spec = parse_field_name(raw_field_name);
let field_id = match &spec.name {
Some(name) => schema.register_field(name),
None => CHILD_FIELD,
};
let mut node_types = type_refs
.clone()
.into_vec()
.into_iter()
.map(|type_ref| {
let (kind, named) = resolve_type_ref_pair(&type_ref, &named_types, &unnamed_types);
crate::schema::NodeType { kind, named }
})
.collect::<Vec<_>>();
node_types.sort_by(|a, b| a.kind.cmp(&b.kind).then(a.named.cmp(&b.named)));
node_types.dedup_by(|a, b| a.kind == b.kind && a.named == b.named);
schema.set_field_types(parent_kind, field_id, node_types);
schema.set_field_cardinality(
parent_kind,
field_id,
crate::schema::FieldCardinality {
multiple: spec.multiple,
required: spec.required,
},
);
}
}
}
pub fn schema_from_yaml(yaml_input: &str) -> Result<crate::schema::Schema, String> {
let mut schema = crate::schema::Schema::new();
extend_schema_from_yaml(&mut schema, yaml_input)?;
Ok(schema)
}
// ---------------------------------------------------------------------------
// JSON → YAML conversion
// ---------------------------------------------------------------------------
/// JSON node-types structures (mirrors tree-sitter's format).
#[derive(Deserialize)]
struct JsonNodeInfo {
#[serde(rename = "type")]
kind: String,
named: bool,
#[serde(default)]
fields: BTreeMap<String, JsonFieldInfo>,
children: Option<JsonFieldInfo>,
#[serde(default)]
subtypes: Vec<JsonNodeType>,
}
#[derive(Deserialize)]
struct JsonNodeType {
#[serde(rename = "type")]
kind: String,
named: bool,
}
#[derive(Deserialize)]
struct JsonFieldInfo {
multiple: bool,
required: bool,
types: Vec<JsonNodeType>,
}
/// Convert a tree-sitter node-types.json string to the YAML format.
pub fn convert_from_json(json_input: &str) -> Result<String, String> {
let nodes: Vec<JsonNodeInfo> =
serde_json::from_str(json_input).map_err(|e| format!("Failed to parse JSON: {e}"))?;
// Collect all named and unnamed types for disambiguation decisions.
let mut all_named: BTreeSet<String> = BTreeSet::new();
let mut all_unnamed: BTreeSet<String> = BTreeSet::new();
for node in &nodes {
if node.named {
all_named.insert(node.kind.clone());
} else {
all_unnamed.insert(node.kind.clone());
}
}
let mut supertypes: BTreeMap<String, Vec<JsonNodeType>> = BTreeMap::new();
let mut named: BTreeMap<String, Option<BTreeMap<String, JsonFieldInfo>>> = BTreeMap::new();
let mut unnamed: Vec<String> = Vec::new();
for node in nodes {
if !node.named {
unnamed.push(node.kind);
continue;
}
if !node.subtypes.is_empty() {
supertypes.insert(node.kind, node.subtypes);
continue;
}
if node.fields.is_empty() && node.children.is_none() {
// Leaf token
named.insert(node.kind, None);
} else {
let mut fields = BTreeMap::new();
for (name, info) in node.fields {
fields.insert(name, info);
}
if let Some(children) = node.children {
fields.insert("$children".to_string(), children);
}
named.insert(node.kind, Some(fields));
}
}
// Now emit YAML
let mut out = String::new();
// Supertypes
if !supertypes.is_empty() {
writeln!(out, "supertypes:").unwrap();
for (name, members) in &supertypes {
writeln!(out, " {name}:").unwrap();
for member in members {
let ref_str = format_type_ref(&member.kind, member.named, &all_named, &all_unnamed);
writeln!(out, " - {ref_str}").unwrap();
}
}
writeln!(out).unwrap();
}
// Named
if !named.is_empty() {
writeln!(out, "named:").unwrap();
for (name, fields_opt) in &named {
match fields_opt {
None => {
writeln!(out, " {name}:").unwrap();
}
Some(fields) => {
writeln!(out, " {name}:").unwrap();
for (field_name, info) in fields {
let suffix = field_suffix(info.multiple, info.required);
let yaml_name = if field_name == "$children" {
format!("$children{suffix}")
} else {
format!("{field_name}{suffix}")
};
let type_refs: Vec<String> = info
.types
.iter()
.map(|t| format_type_ref(&t.kind, t.named, &all_named, &all_unnamed))
.collect();
if type_refs.len() == 1 {
writeln!(out, " {yaml_name}: {}", type_refs[0]).unwrap();
} else {
let list = type_refs
.iter()
.map(|s| s.as_str())
.collect::<Vec<_>>()
.join(", ");
writeln!(out, " {yaml_name}: [{list}]").unwrap();
}
}
}
}
}
writeln!(out).unwrap();
}
// Unnamed
if !unnamed.is_empty() {
writeln!(out, "unnamed:").unwrap();
for name in &unnamed {
writeln!(out, " - {}", force_quote(name)).unwrap();
}
}
Ok(out)
}
fn field_suffix(multiple: bool, required: bool) -> &'static str {
match (multiple, required) {
(false, true) => "",
(false, false) => "?",
(true, true) => "+",
(true, false) => "*",
}
}
/// Format a type reference for YAML output. Uses the disambiguation rule:
/// plain string if unambiguous, `{unnamed: name}` if the name exists as both
/// named and unnamed and we need the unnamed interpretation.
fn format_type_ref(
kind: &str,
named: bool,
all_named: &BTreeSet<String>,
_all_unnamed: &BTreeSet<String>,
) -> String {
if named {
quote_yaml(kind)
} else {
let is_also_named = all_named.contains(kind);
if is_also_named {
format!("{{unnamed: {}}}", force_quote(kind))
} else {
force_quote(kind)
}
}
}
/// Always wrap in double quotes. Used for unnamed node references so they're
/// visually distinct from named ones — YAML treats both forms as equivalent strings.
fn force_quote(s: &str) -> String {
format!("\"{}\"", s.replace('\\', "\\\\").replace('"', "\\\""))
}
/// Quote a YAML string value if it contains special characters or could be
/// misinterpreted.
fn quote_yaml(s: &str) -> String {
let needs_quoting = s.is_empty()
|| s.contains(|c: char| {
matches!(
c,
':' | '{'
| '}'
| '['
| ']'
| ','
| '&'
| '*'
| '#'
| '?'
| '|'
| '-'
| '<'
| '>'
| '='
| '!'
| '%'
| '@'
| '`'
| '"'
| '\''
)
})
|| s.starts_with(' ')
|| s.ends_with(' ')
|| s == "true"
|| s == "false"
|| s == "null"
|| s == "yes"
|| s == "no"
|| s.parse::<f64>().is_ok();
if needs_quoting {
format!("\"{}\"", s.replace('\\', "\\\\").replace('"', "\\\""))
} else {
s.to_string()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_basic_conversion() {
let yaml = r#"
supertypes:
_expression:
- assignment
- binary
named:
assignment:
left: _lhs
right: _expression
binary:
left: [_expression, _simple_numeric]
operator: ["!=", "+"]
right: _expression
argument_list:
$children*: [_expression, block_argument]
identifier:
unnamed:
- "!="
- "+"
- "end"
"#;
let json_str = convert(yaml).unwrap();
let result: Vec<serde_json::Value> = serde_json::from_str(&json_str).unwrap();
// Check supertype
let expr = &result[0];
assert_eq!(expr["type"], "_expression");
assert_eq!(expr["named"], true);
assert_eq!(expr["subtypes"].as_array().unwrap().len(), 2);
// Check assignment
let assign = result.iter().find(|n| n["type"] == "assignment").unwrap();
assert_eq!(assign["fields"]["left"]["required"], true);
assert_eq!(assign["fields"]["left"]["multiple"], false);
assert_eq!(assign["fields"]["left"]["types"][0]["type"], "_lhs");
assert_eq!(assign["fields"]["left"]["types"][0]["named"], true);
// Check binary.operator — "!=" and "+" should resolve to unnamed
let binary = result.iter().find(|n| n["type"] == "binary").unwrap();
let op_types = binary["fields"]["operator"]["types"].as_array().unwrap();
assert_eq!(op_types[0]["type"], "!=");
assert_eq!(op_types[0]["named"], false);
assert_eq!(op_types[1]["type"], "+");
assert_eq!(op_types[1]["named"], false);
// Check argument_list has children, not a field
let arg_list = result
.iter()
.find(|n| n["type"] == "argument_list")
.unwrap();
assert!(arg_list.get("children").is_some());
assert_eq!(arg_list["children"]["multiple"], true);
assert_eq!(arg_list["children"]["required"], false);
// Check identifier is a leaf
let ident = result.iter().find(|n| n["type"] == "identifier").unwrap();
assert_eq!(ident["fields"].as_object().unwrap().len(), 0);
// Check unnamed tokens
let end = result.iter().find(|n| n["type"] == "end").unwrap();
assert_eq!(end["named"], false);
}
#[test]
fn test_explicit_unnamed_disambiguation() {
let yaml = r#"
named:
foo:
field: [{unnamed: bar}]
unnamed:
- bar
"#;
let json_str = convert(yaml).unwrap();
let result: Vec<serde_json::Value> = serde_json::from_str(&json_str).unwrap();
let foo = result.iter().find(|n| n["type"] == "foo").unwrap();
assert_eq!(foo["fields"]["field"]["types"][0]["named"], false);
}
#[test]
fn test_field_suffixes() {
let yaml = r#"
named:
test_node:
required_single: foo
optional_single?: foo
required_multiple+: foo
optional_multiple*: foo
"#;
let json_str = convert(yaml).unwrap();
let result: Vec<serde_json::Value> = serde_json::from_str(&json_str).unwrap();
let node = result.iter().find(|n| n["type"] == "test_node").unwrap();
let fields = node["fields"].as_object().unwrap();
assert_eq!(fields["required_single"]["required"], true);
assert_eq!(fields["required_single"]["multiple"], false);
assert_eq!(fields["optional_single"]["required"], false);
assert_eq!(fields["optional_single"]["multiple"], false);
assert_eq!(fields["required_multiple"]["required"], true);
assert_eq!(fields["required_multiple"]["multiple"], true);
assert_eq!(fields["optional_multiple"]["required"], false);
assert_eq!(fields["optional_multiple"]["multiple"], true);
}
#[test]
fn test_json_to_yaml() {
let json = r#"[
{"type": "_expression", "named": true, "subtypes": [
{"type": "assignment", "named": true},
{"type": "identifier", "named": true}
]},
{"type": "assignment", "named": true, "fields": {
"left": {"multiple": false, "required": true, "types": [
{"type": "_expression", "named": true}
]},
"right": {"multiple": false, "required": false, "types": [
{"type": "_expression", "named": true}
]}
}, "children": {
"multiple": true, "required": false, "types": [
{"type": "identifier", "named": true}
]
}},
{"type": "identifier", "named": true, "fields": {}},
{"type": "=", "named": false},
{"type": "end", "named": false}
]"#;
let yaml = convert_from_json(json).unwrap();
// Verify key structures are present
assert!(yaml.contains("supertypes:"));
assert!(yaml.contains("_expression:"));
assert!(yaml.contains("named:"));
assert!(yaml.contains("assignment:"));
assert!(yaml.contains("left:"));
assert!(yaml.contains("right?:"));
assert!(yaml.contains("$children*:"));
assert!(yaml.contains("identifier:"));
assert!(yaml.contains("unnamed:"));
assert!(yaml.contains("\"=\""));
assert!(yaml.contains("end"));
}
#[test]
fn test_round_trip() {
let yaml_input = r#"
supertypes:
_expression:
- assignment
- identifier
named:
assignment:
left: _expression
right?: _expression
$children*: identifier
identifier:
unnamed:
- "="
- end
"#;
// YAML → JSON → YAML
let json = convert(yaml_input).unwrap();
let yaml_output = convert_from_json(&json).unwrap();
// YAML → JSON again (should be identical)
let json2 = convert(&yaml_output).unwrap();
let v1: serde_json::Value = serde_json::from_str(&json).unwrap();
let v2: serde_json::Value = serde_json::from_str(&json2).unwrap();
assert_eq!(v1, v2);
}
}

340
shared/yeast-schema/src/schema.rs Normal file
View File

@@ -0,0 +1,340 @@
use std::collections::{BTreeMap, BTreeSet};
use crate::{FieldId, KindId, CHILD_FIELD};
#[derive(Clone, Debug)]
pub struct NodeType {
pub kind: String,
pub named: bool,
}
/// Multiplicity/optionality of a field declaration.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct FieldCardinality {
/// Whether the field may hold more than one child.
pub multiple: bool,
/// Whether at least one child must be present.
pub required: bool,
}
/// A schema defining node kinds and field names for the output AST.
/// Built from a node-types.yml file, independent of any tree-sitter grammar.
///
/// # Memory management
///
/// `register_field`/`register_kind`/`register_unnamed_kind` (and their
/// `_with_id` siblings) use `Box::leak` to obtain `&'static str` names. This
/// is intentional: the `&'static str` names appear pervasively in `Node`,
/// `AstCursor`, query patterns, and the extractor's TRAP output, where
/// adding a lifetime would propagate widely.
///
/// The leak is bounded by the number of distinct kind/field names registered.
/// Schemas are expected to be constructed once per process (e.g. at extractor
/// startup) and reused. Repeated construction in long-running processes will
/// leak memory unboundedly and should be avoided.
#[derive(Clone)]
pub struct Schema {
field_ids: BTreeMap<String, FieldId>,
field_names: BTreeMap<FieldId, &'static str>,
next_field_id: FieldId,
kind_ids: BTreeMap<String, KindId>,
unnamed_kind_ids: BTreeMap<String, KindId>,
kind_names: BTreeMap<KindId, &'static str>,
next_kind_id: KindId,
field_types: BTreeMap<(String, FieldId), Vec<NodeType>>,
field_cardinalities: BTreeMap<(String, FieldId), FieldCardinality>,
supertypes: BTreeMap<String, Vec<NodeType>>,
}
impl Default for Schema {
fn default() -> Self {
Self::new()
}
}
impl Schema {
pub fn new() -> Self {
Self {
field_ids: BTreeMap::new(),
field_names: BTreeMap::new(),
next_field_id: 1, // 0 is reserved
kind_ids: BTreeMap::new(),
unnamed_kind_ids: BTreeMap::new(),
kind_names: BTreeMap::new(),
next_kind_id: 1, // 0 is reserved
field_types: BTreeMap::new(),
field_cardinalities: BTreeMap::new(),
supertypes: BTreeMap::new(),
}
}
/// Register a field name, returning its ID.
/// If already registered, returns the existing ID.
pub fn register_field(&mut self, name: &str) -> FieldId {
if name == "child" {
return CHILD_FIELD;
}
if let Some(&id) = self.field_ids.get(name) {
return id;
}
let id = self.next_field_id;
assert!(id < CHILD_FIELD, "too many fields");
self.next_field_id += 1;
let leaked: &'static str = Box::leak(name.to_string().into_boxed_str());
self.field_ids.insert(name.to_string(), id);
self.field_names.insert(id, leaked);
id
}
/// Register a field name with a specific ID, e.g. when importing IDs
/// from an external source like a tree-sitter language. If the name is
/// already registered (with any ID), nothing is changed and the
/// existing ID is returned.
pub fn register_field_with_id(&mut self, name: &str, id: FieldId) -> FieldId {
if name == "child" {
return CHILD_FIELD;
}
if let Some(&existing) = self.field_ids.get(name) {
return existing;
}
assert!(id < CHILD_FIELD, "too many fields");
let leaked: &'static str = Box::leak(name.to_string().into_boxed_str());
self.field_ids.insert(name.to_string(), id);
self.field_names.insert(id, leaked);
if id >= self.next_field_id {
self.next_field_id = id + 1;
}
id
}
/// Register a named node kind name, returning its ID.
/// If already registered, returns the existing ID.
pub fn register_kind(&mut self, name: &str) -> KindId {
if let Some(&id) = self.kind_ids.get(name) {
return id;
}
let id = self.next_kind_id;
self.next_kind_id += 1;
let leaked: &'static str = Box::leak(name.to_string().into_boxed_str());
self.kind_ids.insert(name.to_string(), id);
self.kind_names.insert(id, leaked);
id
}
/// Register a named node kind with a specific ID, e.g. when importing
/// IDs from a tree-sitter language. If the name is already registered,
/// nothing is changed and the existing ID is returned.
pub fn register_kind_with_id(&mut self, name: &str, id: KindId) -> KindId {
if let Some(&existing) = self.kind_ids.get(name) {
return existing;
}
let leaked: &'static str = Box::leak(name.to_string().into_boxed_str());
self.kind_ids.insert(name.to_string(), id);
self.kind_names.insert(id, leaked);
if id >= self.next_kind_id {
self.next_kind_id = id + 1;
}
id
}
/// Register an unnamed token kind (e.g. `"="`, `"end"`), returning its ID.
/// If already registered, returns the existing ID.
pub fn register_unnamed_kind(&mut self, name: &str) -> KindId {
if let Some(&id) = self.unnamed_kind_ids.get(name) {
return id;
}
let id = self.next_kind_id;
self.next_kind_id += 1;
let leaked: &'static str = Box::leak(name.to_string().into_boxed_str());
self.unnamed_kind_ids.insert(name.to_string(), id);
self.kind_names.insert(id, leaked);
id
}
/// Register an unnamed token kind with a specific ID. If the name is
/// already registered as unnamed, nothing is changed and the existing
/// ID is returned.
pub fn register_unnamed_kind_with_id(&mut self, name: &str, id: KindId) -> KindId {
if let Some(&existing) = self.unnamed_kind_ids.get(name) {
return existing;
}
let leaked: &'static str = Box::leak(name.to_string().into_boxed_str());
self.unnamed_kind_ids.insert(name.to_string(), id);
self.kind_names.insert(id, leaked);
if id >= self.next_kind_id {
self.next_kind_id = id + 1;
}
id
}
/// Track a name for a kind ID without registering it as named or
/// unnamed. Useful when importing tree-sitter ID tables that may
/// contain duplicate IDs across the named/unnamed split.
pub fn record_kind_name(&mut self, id: KindId, name: &'static str) {
self.kind_names.entry(id).or_insert(name);
if id >= self.next_kind_id {
self.next_kind_id = id + 1;
}
}
pub fn field_id_for_name(&self, name: &str) -> Option<FieldId> {
if name == "child" {
return Some(CHILD_FIELD);
}
self.field_ids.get(name).copied()
}
pub fn field_name_for_id(&self, id: FieldId) -> Option<&'static str> {
if id == CHILD_FIELD {
return Some("child");
}
self.field_names.get(&id).copied()
}
pub fn id_for_node_kind(&self, kind: &str) -> Option<KindId> {
self.kind_ids.get(kind).copied()
}
pub fn id_for_unnamed_node_kind(&self, kind: &str) -> Option<KindId> {
self.unnamed_kind_ids.get(kind).copied()
}
/// Has `kind` been registered as a named kind (concrete node or
/// supertype)?
pub fn has_named_kind(&self, kind: &str) -> bool {
self.id_for_node_kind(kind).is_some()
}
/// Has `kind` been registered as an unnamed token kind?
pub fn has_unnamed_kind(&self, kind: &str) -> bool {
self.id_for_unnamed_node_kind(kind).is_some()
}
/// Is `field_name` declared as a field on `parent_kind`?
/// `field_name == None` checks for the implicit unfielded slot
/// (`$children`/`CHILD_FIELD`).
pub fn has_field(&self, parent_kind: &str, field_name: Option<&str>) -> bool {
let field_id = match field_name {
Some(name) => match self.field_id_for_name(name) {
Some(id) => id,
None => return false,
},
None => CHILD_FIELD,
};
self.field_types(parent_kind, field_id).is_some()
}
pub fn node_kind_for_id(&self, id: KindId) -> Option<&'static str> {
self.kind_names.get(&id).copied()
}
pub fn set_field_types(
&mut self,
parent_kind: &str,
field_id: FieldId,
node_types: Vec<NodeType>,
) {
self.field_types
.insert((parent_kind.to_string(), field_id), node_types);
}
pub fn field_types(
&self,
parent_kind: &str,
field_id: FieldId,
) -> Option<&Vec<NodeType>> {
self.field_types
.get(&(parent_kind.to_string(), field_id))
}
pub fn set_field_cardinality(
&mut self,
parent_kind: &str,
field_id: FieldId,
cardinality: FieldCardinality,
) {
self.field_cardinalities
.insert((parent_kind.to_string(), field_id), cardinality);
}
/// Returns the declared cardinality for a field, if known.
pub fn field_cardinality(
&self,
parent_kind: &str,
field_id: FieldId,
) -> Option<FieldCardinality> {
self.field_cardinalities
.get(&(parent_kind.to_string(), field_id))
.copied()
}
/// Returns an iterator over all `(field_id, field_name)` pairs that are
/// declared as required (`required: true`) for the given `parent_kind`.
pub fn required_fields_for_kind<'a>(
&'a self,
parent_kind: &'a str,
) -> impl Iterator<Item = (FieldId, Option<&'static str>)> + 'a {
self.field_cardinalities
.iter()
.filter(move |((kind, _), card)| kind == parent_kind && card.required)
.map(move |((_, field_id), _)| {
let name = self.field_name_for_id(*field_id);
(*field_id, name)
})
}
pub fn set_supertype_members(&mut self, supertype: &str, node_types: Vec<NodeType>) {
self.supertypes.insert(supertype.to_string(), node_types);
}
/// Returns the declared members of a supertype, if known.
pub fn supertype_members(&self, supertype: &str) -> Option<&Vec<NodeType>> {
self.supertypes.get(supertype)
}
/// Is `kind` a known supertype (an abstract grouping)?
pub fn is_supertype(&self, kind: &str) -> bool {
self.supertypes.contains_key(kind)
}
fn allows_node(
&self,
node_type: &NodeType,
node_kind: &str,
node_named: bool,
active: &mut BTreeSet<String>,
) -> bool {
if node_type.kind == node_kind && node_type.named == node_named {
return true;
}
if !node_type.named {
return false;
}
let Some(members) = self.supertypes.get(&node_type.kind) else {
return false;
};
if !active.insert(node_type.kind.clone()) {
return false;
}
let matched = members
.iter()
.any(|member| self.allows_node(member, node_kind, node_named, active));
active.remove(&node_type.kind);
matched
}
pub fn node_matches_types(
&self,
node_kind: &str,
node_named: bool,
node_types: &[NodeType],
) -> bool {
node_types.iter().any(|node_type| {
self.allows_node(node_type, node_kind, node_named, &mut BTreeSet::new())
})
}
}

4
shared/yeast/BUILD.bazel
View File

@@ -14,5 +14,7 @@ rust_library(
"//shared/yeast-macros",
],
visibility = ["//visibility:public"],
deps = all_crate_deps(),
deps = all_crate_deps() + [
"//shared/yeast-schema",
],
)

1
shared/yeast/Cargo.toml
View File

@@ -10,6 +10,7 @@ serde_json = "1.0.108"
serde_yaml = "0.9"
tree-sitter = ">= 0.23.0"
yeast-macros = { path = "../yeast-macros" }
yeast-schema = { path = "../yeast-schema" }
tree-sitter-ruby = "0.23"
tree-sitter-python = "0.23"

17
shared/yeast/src/lib.rs
View File

@@ -43,8 +43,13 @@ impl From<Id> for usize {
}
/// Field and Kind ids are provided by tree-sitter
type FieldId = u16;
type KindId = u16;
type FieldId = yeast_schema::FieldId;
type KindId = yeast_schema::KindId;
/// Sentinel field id used to mean "the implicit unfielded slot".
/// Re-exported from `yeast-schema` so the runtime and the schema share a
/// single value.
pub use yeast_schema::CHILD_FIELD;
/// Trait for values that can be appended to a field's id list inside a
/// `tree!`/`trees!`/`rule!` template (in `{expr}` placeholders).
@@ -148,8 +153,6 @@ impl<T: YeastSourceRange + ?Sized> YeastSourceRange for &T {
}
}
pub const CHILD_FIELD: u16 = u16::MAX;
#[derive(Debug)]
pub struct AstCursor<'a> {
ast: &'a Ast,
@@ -295,7 +298,7 @@ impl std::fmt::Debug for Ast {
impl Ast {
/// Construct an AST from a TS tree
pub fn from_tree(language: tree_sitter::Language, tree: &tree_sitter::Tree) -> Self {
let schema = schema::Schema::from_language(&language);
let schema = schema::from_language(&language);
Self::from_tree_with_schema(schema, tree, &language)
}
@@ -1220,7 +1223,7 @@ impl<C> DesugaringConfig<C> {
pub fn build_schema(&self, language: &tree_sitter::Language) -> Result<schema::Schema, String> {
match self.output_node_types_yaml {
Some(yaml) => node_types_yaml::schema_from_yaml_with_language(yaml, language),
None => Ok(schema::Schema::from_language(language)),
None => Ok(schema::from_language(language)),
}
}
}
@@ -1234,7 +1237,7 @@ pub struct Runner<'a, C = ()> {
impl<'a, C> Runner<'a, C> {
/// Create a runner using the input grammar's schema for output.
pub fn new(language: tree_sitter::Language, phases: &'a [Phase<C>]) -> Self {
let schema = schema::Schema::from_language(&language);
let schema = schema::from_language(&language);
Self {
language,
schema,

773
shared/yeast/src/node_types_yaml.rs
View File

@@ -1,767 +1,22 @@
/// Converts a YAML node-types file to the tree-sitter `node-types.json` format.
///
/// # YAML format
///
/// ```yaml
/// supertypes:
/// _expression:
/// - assignment
/// - binary
///
/// named:
/// assignment:
/// left: _lhs
/// right: _expression
/// identifier:
///
/// unnamed:
/// - "+"
/// - "end"
/// ```
///
/// See the crate-level docs for the full format specification.
use std::collections::{BTreeMap, BTreeSet};
use std::fmt::Write;
//! YAML/JSON node-types loaders for YEAST.
//!
//! The pure YAML/JSON conversion routines live in [`yeast_schema::node_types_yaml`].
//! This module re-exports them and adds the tree-sitter-aware adapter
//! [`schema_from_yaml_with_language`].
use crate::CHILD_FIELD;
use serde::Deserialize;
use serde_json::json;
pub use yeast_schema::node_types_yaml::{
convert, convert_from_json, extend_schema_from_yaml, schema_from_yaml,
};
/// Top-level YAML structure.
#[derive(Deserialize, Default)]
struct YamlNodeTypes {
#[serde(default)]
supertypes: BTreeMap<String, Vec<TypeRef>>,
#[serde(default)]
named: BTreeMap<String, Option<BTreeMap<String, TypeRefOrList>>>,
#[serde(default)]
unnamed: Vec<String>,
}
/// A reference to a node type. Can be:
/// - a plain string (resolved by looking up named vs unnamed)
/// - a map `{unnamed: "name"}` to force unnamed interpretation
#[derive(Deserialize, Debug, Clone)]
#[serde(untagged)]
enum TypeRef {
Name(String),
Explicit { unnamed: String },
}
/// A field value: either a single type ref or a list of them.
#[derive(Deserialize, Debug, Clone)]
#[serde(untagged)]
enum TypeRefOrList {
Single(TypeRef),
List(Vec<TypeRef>),
}
impl TypeRefOrList {
fn into_vec(self) -> Vec<TypeRef> {
match self {
TypeRefOrList::Single(t) => vec![t],
TypeRefOrList::List(v) => v,
}
}
}
/// Parsed field name: base name + multiplicity markers.
struct FieldSpec {
name: Option<String>, // None for $children
multiple: bool,
required: bool,
}
fn parse_field_name(raw: &str) -> FieldSpec {
let is_children =
raw == "$children" || raw == "$children?" || raw == "$children*" || raw == "$children+";
let suffix = raw.chars().last().filter(|c| matches!(c, '?' | '*' | '+'));
let (multiple, required) = match suffix {
Some('?') => (false, false),
Some('*') => (true, false),
Some('+') => (true, true),
_ => (false, true), // bare field name = required, single
};
let name = if is_children {
None
} else {
let base = raw.trim_end_matches(['?', '*', '+']);
Some(base.to_string())
};
FieldSpec {
name,
multiple,
required,
}
}
/// Resolve a TypeRef to a (type, named) pair, given the sets of known named
/// and unnamed types.
fn resolve_type_ref_pair(
type_ref: &TypeRef,
named_types: &BTreeSet<String>,
unnamed_types: &BTreeSet<String>,
) -> (String, bool) {
match type_ref {
TypeRef::Explicit { unnamed } => (unnamed.clone(), false),
TypeRef::Name(name) => {
let is_named = named_types.contains(name);
let is_unnamed = unnamed_types.contains(name);
if is_named && is_unnamed {
(name.clone(), true)
} else if is_unnamed {
(name.clone(), false)
} else {
(name.clone(), true)
}
}
}
}
/// Resolve a TypeRef to a {type, named} JSON record, given the sets of known named
/// and unnamed types.
fn resolve_type_ref(
type_ref: &TypeRef,
named_types: &BTreeSet<String>,
unnamed_types: &BTreeSet<String>,
) -> serde_json::Value {
let (kind, named) = resolve_type_ref_pair(type_ref, named_types, unnamed_types);
json!({"type": kind, "named": named})
}
/// Convert YAML string to node-types JSON string.
pub fn convert(yaml_input: &str) -> Result<String, String> {
let yaml: YamlNodeTypes =
serde_yaml::from_str(yaml_input).map_err(|e| format!("Failed to parse YAML: {e}"))?;
// Build the sets of known named and unnamed types for resolution.
let mut named_types = BTreeSet::new();
for name in yaml.supertypes.keys() {
named_types.insert(name.clone());
}
for name in yaml.named.keys() {
named_types.insert(name.clone());
}
let unnamed_types: BTreeSet<String> = yaml.unnamed.iter().cloned().collect();
let mut output = Vec::new();
// 1. Supertypes
for (name, members) in &yaml.supertypes {
let subtypes: Vec<_> = members
.iter()
.map(|m| resolve_type_ref(m, &named_types, &unnamed_types))
.collect();
output.push(json!({
"type": name,
"named": true,
"subtypes": subtypes,
}));
}
// 2. Named nodes
for (name, fields_opt) in &yaml.named {
let fields_map = match fields_opt {
None => {
// Leaf token: no fields, no children, no subtypes
output.push(json!({
"type": name,
"named": true,
"fields": {},
}));
continue;
}
Some(m) if m.is_empty() => {
output.push(json!({
"type": name,
"named": true,
"fields": {},
}));
continue;
}
Some(m) => m,
};
let mut json_fields = serde_json::Map::new();
let mut json_children: Option<serde_json::Value> = None;
for (raw_field_name, type_refs) in fields_map {
let spec = parse_field_name(raw_field_name);
let types: Vec<_> = type_refs
.clone()
.into_vec()
.iter()
.map(|t| resolve_type_ref(t, &named_types, &unnamed_types))
.collect();
// Cloning to make the borrow checker happy
let field_info = json!({
"multiple": spec.multiple,
"required": spec.required,
"types": types,
});
if spec.name.is_none() {
// $children
json_children = Some(field_info);
} else {
json_fields.insert(spec.name.unwrap(), field_info);
}
}
let mut entry = json!({
"type": name,
"named": true,
"fields": json_fields,
});
if let Some(children) = json_children {
entry
.as_object_mut()
.unwrap()
.insert("children".to_string(), children);
}
output.push(entry);
}
// 3. Unnamed tokens
for name in &yaml.unnamed {
output.push(json!({
"type": name,
"named": false,
}));
}
serde_json::to_string_pretty(&output).map_err(|e| format!("Failed to serialize JSON: {e}"))
}
/// Apply YAML node-type definitions to a mutable Schema.
/// Registers all types, fields, and allowed types from the YAML into the schema.
fn apply_yaml_to_schema(yaml: &YamlNodeTypes, schema: &mut crate::schema::Schema) {
// Register all supertypes as node kinds
for name in yaml.supertypes.keys() {
schema.register_kind(name);
}
// Register named node kinds and their fields
for (name, fields_opt) in &yaml.named {
schema.register_kind(name);
if let Some(fields) = fields_opt {
for raw_field_name in fields.keys() {
let spec = parse_field_name(raw_field_name);
if let Some(field_name) = &spec.name {
schema.register_field(field_name);
}
}
}
}
// Register unnamed tokens as node kinds
for name in &yaml.unnamed {
schema.register_unnamed_kind(name);
}
let mut named_types = BTreeSet::new();
for name in yaml.supertypes.keys() {
named_types.insert(name.clone());
}
for name in yaml.named.keys() {
named_types.insert(name.clone());
}
let unnamed_types: BTreeSet<String> = yaml.unnamed.iter().cloned().collect();
for (supertype, members) in &yaml.supertypes {
let node_types = members
.iter()
.map(|m| {
let (kind, named) = resolve_type_ref_pair(m, &named_types, &unnamed_types);
crate::schema::NodeType { kind, named }
})
.collect();
schema.set_supertype_members(supertype, node_types);
}
// Register allowed field child types for type checking.
for (parent_kind, fields_opt) in &yaml.named {
let Some(fields) = fields_opt else {
continue;
};
for (raw_field_name, type_refs) in fields {
let spec = parse_field_name(raw_field_name);
let field_id = match &spec.name {
Some(name) => schema.register_field(name),
None => CHILD_FIELD,
};
let mut node_types = type_refs
.clone()
.into_vec()
.into_iter()
.map(|type_ref| {
let (kind, named) =
resolve_type_ref_pair(&type_ref, &named_types, &unnamed_types);
crate::schema::NodeType { kind, named }
})
.collect::<Vec<_>>();
node_types.sort_by(|a, b| a.kind.cmp(&b.kind).then(a.named.cmp(&b.named)));
node_types.dedup_by(|a, b| a.kind == b.kind && a.named == b.named);
schema.set_field_types(parent_kind, field_id, node_types);
schema.set_field_cardinality(
parent_kind,
field_id,
crate::schema::FieldCardinality {
multiple: spec.multiple,
required: spec.required,
},
);
}
}
}
pub fn schema_from_yaml(yaml_input: &str) -> Result<crate::schema::Schema, String> {
let yaml: YamlNodeTypes =
serde_yaml::from_str(yaml_input).map_err(|e| format!("Failed to parse YAML: {e}"))?;
let mut schema = crate::schema::Schema::new();
apply_yaml_to_schema(&yaml, &mut schema);
Ok(schema)
}
/// Build a Schema from a YAML string, extending a tree-sitter Language.
/// The Schema inherits all field/kind names from the Language, plus any
/// additional ones defined in the YAML.
/// Build a Schema from a YAML string, layered on top of a tree-sitter
/// `Language`. The Schema inherits all field/kind names from the language
/// (preserving the language's IDs), plus any additional ones defined in
/// the YAML.
pub fn schema_from_yaml_with_language(
yaml_input: &str,
language: &tree_sitter::Language,
) -> Result<crate::schema::Schema, String> {
let yaml: YamlNodeTypes =
serde_yaml::from_str(yaml_input).map_err(|e| format!("Failed to parse YAML: {e}"))?;
let mut schema = crate::schema::Schema::from_language(language);
apply_yaml_to_schema(&yaml, &mut schema);
let mut schema = crate::schema::from_language(language);
extend_schema_from_yaml(&mut schema, yaml_input)?;
Ok(schema)
}
// ---------------------------------------------------------------------------
// JSON → YAML conversion
// ---------------------------------------------------------------------------
/// JSON node-types structures (mirrors tree-sitter's format).
#[derive(Deserialize)]
struct JsonNodeInfo {
#[serde(rename = "type")]
kind: String,
named: bool,
#[serde(default)]
fields: BTreeMap<String, JsonFieldInfo>,
children: Option<JsonFieldInfo>,
#[serde(default)]
subtypes: Vec<JsonNodeType>,
}
#[derive(Deserialize)]
struct JsonNodeType {
#[serde(rename = "type")]
kind: String,
named: bool,
}
#[derive(Deserialize)]
struct JsonFieldInfo {
multiple: bool,
required: bool,
types: Vec<JsonNodeType>,
}
/// Convert a tree-sitter node-types.json string to the YAML format.
pub fn convert_from_json(json_input: &str) -> Result<String, String> {
let nodes: Vec<JsonNodeInfo> =
serde_json::from_str(json_input).map_err(|e| format!("Failed to parse JSON: {e}"))?;
// Collect all named and unnamed types for disambiguation decisions.
let mut all_named: BTreeSet<String> = BTreeSet::new();
let mut all_unnamed: BTreeSet<String> = BTreeSet::new();
for node in &nodes {
if node.named {
all_named.insert(node.kind.clone());
} else {
all_unnamed.insert(node.kind.clone());
}
}
let mut supertypes: BTreeMap<String, Vec<JsonNodeType>> = BTreeMap::new();
let mut named: BTreeMap<String, Option<BTreeMap<String, JsonFieldInfo>>> = BTreeMap::new();
let mut unnamed: Vec<String> = Vec::new();
for node in nodes {
if !node.named {
unnamed.push(node.kind);
continue;
}
if !node.subtypes.is_empty() {
supertypes.insert(node.kind, node.subtypes);
continue;
}
if node.fields.is_empty() && node.children.is_none() {
// Leaf token
named.insert(node.kind, None);
} else {
let mut fields = BTreeMap::new();
for (name, info) in node.fields {
fields.insert(name, info);
}
if let Some(children) = node.children {
fields.insert("$children".to_string(), children);
}
named.insert(node.kind, Some(fields));
}
}
// Now emit YAML
let mut out = String::new();
// Supertypes
if !supertypes.is_empty() {
writeln!(out, "supertypes:").unwrap();
for (name, members) in &supertypes {
writeln!(out, " {name}:").unwrap();
for member in members {
let ref_str = format_type_ref(&member.kind, member.named, &all_named, &all_unnamed);
writeln!(out, " - {ref_str}").unwrap();
}
}
writeln!(out).unwrap();
}
// Named
if !named.is_empty() {
writeln!(out, "named:").unwrap();
for (name, fields_opt) in &named {
match fields_opt {
None => {
writeln!(out, " {name}:").unwrap();
}
Some(fields) => {
writeln!(out, " {name}:").unwrap();
for (field_name, info) in fields {
let suffix = field_suffix(info.multiple, info.required);
let yaml_name = if field_name == "$children" {
format!("$children{suffix}")
} else {
format!("{field_name}{suffix}")
};
let type_refs: Vec<String> = info
.types
.iter()
.map(|t| format_type_ref(&t.kind, t.named, &all_named, &all_unnamed))
.collect();
if type_refs.len() == 1 {
writeln!(out, " {yaml_name}: {}", type_refs[0]).unwrap();
} else {
let list = type_refs
.iter()
.map(|s| s.as_str())
.collect::<Vec<_>>()
.join(", ");
writeln!(out, " {yaml_name}: [{list}]").unwrap();
}
}
}
}
}
writeln!(out).unwrap();
}
// Unnamed
if !unnamed.is_empty() {
writeln!(out, "unnamed:").unwrap();
for name in &unnamed {
writeln!(out, " - {}", force_quote(name)).unwrap();
}
}
Ok(out)
}
fn field_suffix(multiple: bool, required: bool) -> &'static str {
match (multiple, required) {
(false, true) => "",
(false, false) => "?",
(true, true) => "+",
(true, false) => "*",
}
}
/// Format a type reference for YAML output. Uses the disambiguation rule:
/// plain string if unambiguous, `{unnamed: name}` if the name exists as both
/// named and unnamed and we need the unnamed interpretation.
fn format_type_ref(
kind: &str,
named: bool,
all_named: &BTreeSet<String>,
_all_unnamed: &BTreeSet<String>,
) -> String {
if named {
quote_yaml(kind)
} else {
let is_also_named = all_named.contains(kind);
if is_also_named {
format!("{{unnamed: {}}}", force_quote(kind))
} else {
force_quote(kind)
}
}
}
/// Always wrap in double quotes. Used for unnamed node references so they're
/// visually distinct from named ones — YAML treats both forms as equivalent strings.
fn force_quote(s: &str) -> String {
format!("\"{}\"", s.replace('\\', "\\\\").replace('"', "\\\""))
}
/// Quote a YAML string value if it contains special characters or could be
/// misinterpreted.
fn quote_yaml(s: &str) -> String {
let needs_quoting = s.is_empty()
|| s.contains(|c: char| {
matches!(
c,
':' | '{'
| '}'
| '['
| ']'
| ','
| '&'
| '*'
| '#'
| '?'
| '|'
| '-'
| '<'
| '>'
| '='
| '!'
| '%'
| '@'
| '`'
| '"'
| '\''
)
})
|| s.starts_with(' ')
|| s.ends_with(' ')
|| s == "true"
|| s == "false"
|| s == "null"
|| s == "yes"
|| s == "no"
|| s.parse::<f64>().is_ok();
if needs_quoting {
format!("\"{}\"", s.replace('\\', "\\\\").replace('"', "\\\""))
} else {
s.to_string()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_basic_conversion() {
let yaml = r#"
supertypes:
_expression:
- assignment
- binary
named:
assignment:
left: _lhs
right: _expression
binary:
left: [_expression, _simple_numeric]
operator: ["!=", "+"]
right: _expression
argument_list:
$children*: [_expression, block_argument]
identifier:
unnamed:
- "!="
- "+"
- "end"
"#;
let json_str = convert(yaml).unwrap();
let result: Vec<serde_json::Value> = serde_json::from_str(&json_str).unwrap();
// Check supertype
let expr = &result[0];
assert_eq!(expr["type"], "_expression");
assert_eq!(expr["named"], true);
assert_eq!(expr["subtypes"].as_array().unwrap().len(), 2);
// Check assignment
let assign = result.iter().find(|n| n["type"] == "assignment").unwrap();
assert_eq!(assign["fields"]["left"]["required"], true);
assert_eq!(assign["fields"]["left"]["multiple"], false);
assert_eq!(assign["fields"]["left"]["types"][0]["type"], "_lhs");
assert_eq!(assign["fields"]["left"]["types"][0]["named"], true);
// Check binary.operator — "!=" and "+" should resolve to unnamed
let binary = result.iter().find(|n| n["type"] == "binary").unwrap();
let op_types = binary["fields"]["operator"]["types"].as_array().unwrap();
assert_eq!(op_types[0]["type"], "!=");
assert_eq!(op_types[0]["named"], false);
assert_eq!(op_types[1]["type"], "+");
assert_eq!(op_types[1]["named"], false);
// Check argument_list has children, not a field
let arg_list = result
.iter()
.find(|n| n["type"] == "argument_list")
.unwrap();
assert!(arg_list.get("children").is_some());
assert_eq!(arg_list["children"]["multiple"], true);
assert_eq!(arg_list["children"]["required"], false);
// Check identifier is a leaf
let ident = result.iter().find(|n| n["type"] == "identifier").unwrap();
assert_eq!(ident["fields"].as_object().unwrap().len(), 0);
// Check unnamed tokens
let end = result.iter().find(|n| n["type"] == "end").unwrap();
assert_eq!(end["named"], false);
}
#[test]
fn test_explicit_unnamed_disambiguation() {
let yaml = r#"
named:
foo:
field: [{unnamed: bar}]
unnamed:
- bar
"#;
let json_str = convert(yaml).unwrap();
let result: Vec<serde_json::Value> = serde_json::from_str(&json_str).unwrap();
let foo = result.iter().find(|n| n["type"] == "foo").unwrap();
assert_eq!(foo["fields"]["field"]["types"][0]["named"], false);
}
#[test]
fn test_field_suffixes() {
let yaml = r#"
named:
test_node:
required_single: foo
optional_single?: foo
required_multiple+: foo
optional_multiple*: foo
"#;
let json_str = convert(yaml).unwrap();
let result: Vec<serde_json::Value> = serde_json::from_str(&json_str).unwrap();
let node = result.iter().find(|n| n["type"] == "test_node").unwrap();
let fields = node["fields"].as_object().unwrap();
assert_eq!(fields["required_single"]["required"], true);
assert_eq!(fields["required_single"]["multiple"], false);
assert_eq!(fields["optional_single"]["required"], false);
assert_eq!(fields["optional_single"]["multiple"], false);
assert_eq!(fields["required_multiple"]["required"], true);
assert_eq!(fields["required_multiple"]["multiple"], true);
assert_eq!(fields["optional_multiple"]["required"], false);
assert_eq!(fields["optional_multiple"]["multiple"], true);
}
#[test]
fn test_json_to_yaml() {
let json = r#"[
{"type": "_expression", "named": true, "subtypes": [
{"type": "assignment", "named": true},
{"type": "identifier", "named": true}
]},
{"type": "assignment", "named": true, "fields": {
"left": {"multiple": false, "required": true, "types": [
{"type": "_expression", "named": true}
]},
"right": {"multiple": false, "required": false, "types": [
{"type": "_expression", "named": true}
]}
}, "children": {
"multiple": true, "required": false, "types": [
{"type": "identifier", "named": true}
]
}},
{"type": "identifier", "named": true, "fields": {}},
{"type": "=", "named": false},
{"type": "end", "named": false}
]"#;
let yaml = convert_from_json(json).unwrap();
// Verify key structures are present
assert!(yaml.contains("supertypes:"));
assert!(yaml.contains("_expression:"));
assert!(yaml.contains("named:"));
assert!(yaml.contains("assignment:"));
assert!(yaml.contains("left:"));
assert!(yaml.contains("right?:"));
assert!(yaml.contains("$children*:"));
assert!(yaml.contains("identifier:"));
assert!(yaml.contains("unnamed:"));
assert!(yaml.contains("\"=\""));
assert!(yaml.contains("end"));
}
#[test]
fn test_round_trip() {
let yaml_input = r#"
supertypes:
_expression:
- assignment
- identifier
named:
assignment:
left: _expression
right?: _expression
$children*: identifier
identifier:
unnamed:
- "="
- end
"#;
// YAML → JSON → YAML
let json = convert(yaml_input).unwrap();
let yaml_output = convert_from_json(&json).unwrap();
// YAML → JSON again (should be identical)
let json2 = convert(&yaml_output).unwrap();
let v1: serde_json::Value = serde_json::from_str(&json).unwrap();
let v2: serde_json::Value = serde_json::from_str(&json2).unwrap();
assert_eq!(v1, v2);
}
}

315
shared/yeast/src/schema.rs
View File

@@ -1,285 +1,54 @@
use std::collections::{BTreeMap, BTreeSet};
//! YEAST schema types.
//!
//! The schema struct itself lives in the [`yeast_schema`] crate (so it can
//! be shared with the `yeast-macros` proc-macro crate without dragging
//! tree-sitter into proc-macro compiles). This module re-exports its
//! public API and supplies the one tree-sitter-aware adapter the runtime
//! needs: [`from_language`].
use crate::{FieldId, KindId, CHILD_FIELD};
pub use yeast_schema::schema::{FieldCardinality, NodeType, Schema};
#[derive(Clone, Debug)]
pub struct NodeType {
pub kind: String,
pub named: bool,
}
/// Build a [`Schema`] from a tree-sitter language, importing all its
/// known field and kind names so the resulting schema's IDs line up with
/// the language's own IDs (i.e. `field_name_for_id` agrees).
pub fn from_language(language: &tree_sitter::Language) -> Schema {
let mut schema = Schema::new();
/// Multiplicity/optionality of a field declaration.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct FieldCardinality {
/// Whether the field may hold more than one child.
pub multiple: bool,
/// Whether at least one child must be present.
pub required: bool,
}
/// A schema defining node kinds and field names for the output AST.
/// Built from a node-types.yml file, independent of any tree-sitter grammar.
///
/// # Memory management
///
/// `register_field`/`register_kind`/`register_unnamed_kind` use `Box::leak`
/// to obtain `&'static str` names. This is intentional: the `&'static str`
/// names appear pervasively in `Node`, `AstCursor`, query patterns, and the
/// extractor's TRAP output, where adding a lifetime would propagate widely.
///
/// The leak is bounded by the number of distinct kind/field names registered.
/// Schemas are expected to be constructed once per process (e.g. at extractor
/// startup) and reused. Repeated construction in long-running processes will
/// leak memory unboundedly and should be avoided.
#[derive(Clone)]
pub struct Schema {
field_ids: BTreeMap<String, FieldId>,
field_names: BTreeMap<FieldId, &'static str>,
next_field_id: FieldId,
kind_ids: BTreeMap<String, KindId>,
unnamed_kind_ids: BTreeMap<String, KindId>,
kind_names: BTreeMap<KindId, &'static str>,
next_kind_id: KindId,
field_types: BTreeMap<(String, FieldId), Vec<NodeType>>,
field_cardinalities: BTreeMap<(String, FieldId), FieldCardinality>,
supertypes: BTreeMap<String, Vec<NodeType>>,
}
impl Default for Schema {
fn default() -> Self {
Self::new()
}
}
impl Schema {
pub fn new() -> Self {
Self {
field_ids: BTreeMap::new(),
field_names: BTreeMap::new(),
next_field_id: 1, // 0 is reserved
kind_ids: BTreeMap::new(),
unnamed_kind_ids: BTreeMap::new(),
kind_names: BTreeMap::new(),
next_kind_id: 1, // 0 is reserved
field_types: BTreeMap::new(),
field_cardinalities: BTreeMap::new(),
supertypes: BTreeMap::new(),
// Import all field names, preserving tree-sitter's IDs.
for id in 1..=language.field_count() as u16 {
if let Some(name) = language.field_name_for_id(id) {
schema.register_field_with_id(name, id);
}
}
/// Create a schema from a tree-sitter language, importing all its
/// known field and kind names.
pub fn from_language(language: &tree_sitter::Language) -> Self {
let mut schema = Self::new();
// Import all field names, preserving tree-sitter's IDs
for id in 1..=language.field_count() as u16 {
if let Some(name) = language.field_name_for_id(id) {
schema.field_ids.insert(name.to_string(), id);
schema.field_names.insert(id, name);
if id >= schema.next_field_id {
schema.next_field_id = id + 1;
// Import all node kind names, preserving tree-sitter's IDs.
// Track named and unnamed variants separately. For both, prefer the
// canonical ID returned by `id_for_node_kind`, since some languages
// have multiple IDs for the same name (e.g. the reserved error token
// at ID 0 may share a name with a real token).
for id in 0..language.node_kind_count() as u16 {
if let Some(name) = language.node_kind_for_id(id) {
if name.is_empty() {
continue;
}
let is_named = language.node_kind_is_named(id);
if is_named {
let canonical_id = language.id_for_node_kind(name, true);
if canonical_id != 0 && schema.id_for_node_kind(name).is_none() {
schema.register_kind_with_id(name, canonical_id);
}
} else {
let canonical_id = language.id_for_node_kind(name, false);
if canonical_id != 0 && schema.id_for_unnamed_node_kind(name).is_none() {
schema.register_unnamed_kind_with_id(name, canonical_id);
}
}
// Always track the name for any ID we encounter (so
// `node_kind_for_id` works for the literal `id` we saw, even
// when it isn't the canonical one).
schema.record_kind_name(id, name);
}
// Import all node kind names, preserving tree-sitter's IDs.
// Track named and unnamed variants separately. For both named and
// unnamed kinds, use the canonical ID from id_for_node_kind, since
// some languages have multiple IDs for the same name (e.g., the
// reserved error token at ID 0 may share a name with a real token).
for id in 0..language.node_kind_count() as u16 {
if let Some(name) = language.node_kind_for_id(id) {
if !name.is_empty() {
let is_named = language.node_kind_is_named(id);
if is_named {
let canonical_id = language.id_for_node_kind(name, true);
if canonical_id != 0 && !schema.kind_ids.contains_key(name) {
schema.kind_ids.insert(name.to_string(), canonical_id);
schema.kind_names.insert(canonical_id, name);
}
} else {
let canonical_id = language.id_for_node_kind(name, false);
if canonical_id != 0 && !schema.unnamed_kind_ids.contains_key(name) {
schema
.unnamed_kind_ids
.insert(name.to_string(), canonical_id);
schema.kind_names.insert(canonical_id, name);
}
}
// Always track the name for any ID we encounter
schema.kind_names.entry(id).or_insert(name);
if id >= schema.next_kind_id {
schema.next_kind_id = id + 1;
}
}
}
}
schema
}
/// Register a field name, returning its ID.
/// If already registered, returns the existing ID.
pub fn register_field(&mut self, name: &str) -> FieldId {
if name == "child" {
return CHILD_FIELD;
}
if let Some(&id) = self.field_ids.get(name) {
return id;
}
let id = self.next_field_id;
assert!(id < CHILD_FIELD, "too many fields");
self.next_field_id += 1;
let leaked: &'static str = Box::leak(name.to_string().into_boxed_str());
self.field_ids.insert(name.to_string(), id);
self.field_names.insert(id, leaked);
id
}
/// Register a named node kind name, returning its ID.
/// If already registered, returns the existing ID.
pub fn register_kind(&mut self, name: &str) -> KindId {
if let Some(&id) = self.kind_ids.get(name) {
return id;
}
let id = self.next_kind_id;
self.next_kind_id += 1;
let leaked: &'static str = Box::leak(name.to_string().into_boxed_str());
self.kind_ids.insert(name.to_string(), id);
self.kind_names.insert(id, leaked);
id
}
/// Register an unnamed token kind (e.g. `"="`, `"end"`), returning its ID.
/// If already registered, returns the existing ID.
pub fn register_unnamed_kind(&mut self, name: &str) -> KindId {
if let Some(&id) = self.unnamed_kind_ids.get(name) {
return id;
}
let id = self.next_kind_id;
self.next_kind_id += 1;
let leaked: &'static str = Box::leak(name.to_string().into_boxed_str());
self.unnamed_kind_ids.insert(name.to_string(), id);
self.kind_names.insert(id, leaked);
id
}
pub fn field_id_for_name(&self, name: &str) -> Option<FieldId> {
if name == "child" {
return Some(CHILD_FIELD);
}
self.field_ids.get(name).copied()
}
pub fn field_name_for_id(&self, id: FieldId) -> Option<&'static str> {
if id == CHILD_FIELD {
return Some("child");
}
self.field_names.get(&id).copied()
}
pub fn id_for_node_kind(&self, kind: &str) -> Option<KindId> {
self.kind_ids.get(kind).copied()
}
pub fn id_for_unnamed_node_kind(&self, kind: &str) -> Option<KindId> {
self.unnamed_kind_ids.get(kind).copied()
}
pub fn node_kind_for_id(&self, id: KindId) -> Option<&'static str> {
self.kind_names.get(&id).copied()
}
pub fn set_field_types(
&mut self,
parent_kind: &str,
field_id: FieldId,
node_types: Vec<NodeType>,
) {
self.field_types
.insert((parent_kind.to_string(), field_id), node_types);
}
pub fn field_types(&self, parent_kind: &str, field_id: FieldId) -> Option<&Vec<NodeType>> {
self.field_types.get(&(parent_kind.to_string(), field_id))
}
pub fn set_field_cardinality(
&mut self,
parent_kind: &str,
field_id: FieldId,
cardinality: FieldCardinality,
) {
self.field_cardinalities
.insert((parent_kind.to_string(), field_id), cardinality);
}
/// Returns the declared cardinality for a field, if known.
pub fn field_cardinality(
&self,
parent_kind: &str,
field_id: FieldId,
) -> Option<FieldCardinality> {
self.field_cardinalities
.get(&(parent_kind.to_string(), field_id))
.copied()
}
/// Returns an iterator over all `(field_id, field_name)` pairs that are
/// declared as required (`required: true`) for the given `parent_kind`.
pub fn required_fields_for_kind<'a>(
&'a self,
parent_kind: &'a str,
) -> impl Iterator<Item = (FieldId, Option<&'static str>)> + 'a {
self.field_cardinalities
.iter()
.filter(move |((kind, _), card)| kind == parent_kind && card.required)
.map(move |((_, field_id), _)| {
let name = self.field_name_for_id(*field_id);
(*field_id, name)
})
}
pub fn set_supertype_members(&mut self, supertype: &str, node_types: Vec<NodeType>) {
self.supertypes.insert(supertype.to_string(), node_types);
}
fn allows_node(
&self,
node_type: &NodeType,
node_kind: &str,
node_named: bool,
active: &mut BTreeSet<String>,
) -> bool {
if node_type.kind == node_kind && node_type.named == node_named {
return true;
}
if !node_type.named {
return false;
}
let Some(members) = self.supertypes.get(&node_type.kind) else {
return false;
};
if !active.insert(node_type.kind.clone()) {
return false;
}
let matched = members
.iter()
.any(|member| self.allows_node(member, node_kind, node_named, active));
active.remove(&node_type.kind);
matched
}
pub fn node_matches_types(
&self,
node_kind: &str,
node_named: bool,
node_types: &[NodeType],
) -> bool {
node_types.iter().any(|node_type| {
self.allows_node(node_type, node_kind, node_named, &mut BTreeSet::new())
})
}
schema
}