diff --git a/Cargo.lock b/Cargo.lock index 24885fea856..7a9f1966791 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2853,10 +2853,6 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "swift-syntax-rs" version = "0.1.0" -dependencies = [ - "serde_json", - "yeast", -] [[package]] name = "syn" diff --git a/shared/yeast-schema/src/schema.rs b/shared/yeast-schema/src/schema.rs index 4acd14377a4..9c438d7c22e 100644 --- a/shared/yeast-schema/src/schema.rs +++ b/shared/yeast-schema/src/schema.rs @@ -167,6 +167,28 @@ impl Schema { id } + /// Register every kind (named and unnamed) and field *name* from `other` + /// into this schema (idempotent). Ids are assigned in this schema's own id + /// space; existing ids are unchanged. + /// + /// This is used when running desugaring rules over an AST that was built + /// against a different schema (e.g. from an external parser): the rules + /// build output nodes whose kind/field names come from `other`, and those + /// names must resolve in the AST's own schema. Only names are needed — the + /// rule engine resolves kinds/fields by name and does not consult + /// `other`'s field-type or supertype information. + pub fn register_names_from(&mut self, other: &Schema) { + for name in other.kind_ids.keys() { + self.register_kind(name); + } + for name in other.unnamed_kind_ids.keys() { + self.register_unnamed_kind(name); + } + for name in other.field_ids.keys() { + self.register_field(name); + } + } + /// Track a name for a kind ID without registering it as named or /// unnamed. Useful when importing tree-sitter ID tables that may /// contain duplicate IDs across the named/unnamed split. diff --git a/shared/yeast/src/lib.rs b/shared/yeast/src/lib.rs index b2709ab406e..89facfaea99 100644 --- a/shared/yeast/src/lib.rs +++ b/shared/yeast/src/lib.rs @@ -524,10 +524,15 @@ impl Ast { self.schema.register_field(name) } - fn union_source_range_of_children( - &self, - fields: &BTreeMap>, - ) -> Option { + /// Register every kind and field name from `schema` into this AST's schema + /// (idempotent). Used before desugaring an externally-built AST so that + /// rules can build output nodes whose kind/field names come from the + /// desugarer's output schema. + pub fn register_names_from_schema(&mut self, schema: &schema::Schema) { + self.schema.register_names_from(schema); + } + + fn union_source_range_of_children(&self, fields: &BTreeMap>) -> Option { let mut start_byte: Option = None; let mut end_byte: Option = None; let mut start_point = Point { row: 0, column: 0 }; @@ -1448,6 +1453,18 @@ impl<'a, C: Clone + Default> Runner<'a, C> { let mut user_ctx = C::default(); self.run_with_ctx(input, &mut user_ctx) } + + /// Run all phases over an already-built `ast`, using the default context + /// (`C::default()`). Unlike [`run_from_tree`](Self::run_from_tree), the AST + /// is supplied by the caller (e.g. built from an external parser's output) + /// rather than constructed from a tree-sitter tree. The caller is + /// responsible for ensuring the AST's schema knows any output kind/field + /// names the rules will build (see [`Ast::register_names_from_schema`]). + pub fn run_from_ast(&self, mut ast: Ast) -> Result { + let mut user_ctx = C::default(); + self.run_phases(&mut ast, &mut user_ctx)?; + Ok(ast) + } } // --------------------------------------------------------------------------- @@ -1470,6 +1487,12 @@ pub trait Desugarer: Send + Sync { /// Parse `tree` against `source` and run the desugaring pipeline. /// Each call constructs a fresh default user context internally. fn run_from_tree(&self, tree: &tree_sitter::Tree, source: &[u8]) -> Result; + + /// Run the desugaring pipeline over an already-built `ast` (e.g. produced + /// by an external parser rather than tree-sitter). The desugarer ensures + /// the AST's schema knows its output kind/field names before running the + /// rules. Each call constructs a fresh default user context internally. + fn run_from_ast(&self, ast: Ast) -> Result; } /// A concrete [`Desugarer`] backed by a [`DesugaringConfig`] for a @@ -1507,4 +1530,12 @@ impl Desugarer for ConcreteDesugarer let runner = Runner::with_schema(self.language.clone(), &self.schema, &self.config.phases); runner.run_from_tree(tree, source) } + + fn run_from_ast(&self, mut ast: Ast) -> Result { + // The AST was built against its own (external) schema; make sure the + // output kind/field names the rules build are resolvable in it. + ast.register_names_from_schema(&self.schema); + let runner = Runner::with_schema(self.language.clone(), &self.schema, &self.config.phases); + runner.run_from_ast(ast) + } } diff --git a/shared/yeast/tests/test.rs b/shared/yeast/tests/test.rs index 3a24709dd9f..756e219bd89 100644 --- a/shared/yeast/tests/test.rs +++ b/shared/yeast/tests/test.rs @@ -266,6 +266,59 @@ fn test_query_match() { assert!(captures.get_var("right").is_ok()); } +#[test] +fn test_run_from_ast_desugars_hand_built_tree() { + use std::collections::BTreeMap; + + // Output schema for the desugared tree. Its kind/field names must become + // resolvable in the hand-built AST's schema for the rule to build them. + let schema_yaml = r#" +named: + assignment: + left: leaf + leaf: +"#; + + // A rule over an *input* kind (`wrapper`) that is not in the output schema, + // rewriting to an output `assignment` node. + let rules: Vec = vec![yeast::rule!( + (wrapper) + => + (assignment left: (leaf "lit")) + )]; + + let lang: tree_sitter::Language = tree_sitter_ruby::LANGUAGE.into(); + let config = DesugaringConfig::<()>::new() + .add_phase("test", PhaseKind::OneShot, rules) + .with_output_node_types_yaml(schema_yaml); + let desugarer = ConcreteDesugarer::new(lang, config).unwrap(); + + // Build the input AST by hand, as an external parser adapter would. The + // schema starts empty and gains the `wrapper` input kind on the fly. + let mut ast = Ast::with_schema(yeast::schema::Schema::new()); + let wrapper_kind = ast.register_kind("wrapper"); + let root = ast.create_node_with_range( + wrapper_kind, + NodeContent::DynamicString(String::new()), + BTreeMap::new(), + true, + None, + ); + ast.set_root(root); + + // Desugaring the hand-built AST applies the rule, producing `assignment` + // even though the AST was built against a schema with no output kinds. + let out = desugarer + .run_from_ast(ast) + .expect("run_from_ast should succeed"); + let out_root = out.get_node(out.get_root()).expect("root exists"); + assert_eq!(out_root.kind_name(), "assignment"); + let dump = dump_ast(&out, out.get_root(), ""); + assert!(dump.contains("assignment"), "unexpected dump: {dump}"); + assert!(dump.contains("left"), "unexpected dump: {dump}"); + assert!(dump.contains("leaf"), "unexpected dump: {dump}"); +} + #[test] fn test_query_no_match() { let runner: Runner = Runner::new(tree_sitter_ruby::LANGUAGE.into(), &[]); diff --git a/unified/extractor/src/languages/mod.rs b/unified/extractor/src/languages/mod.rs index 20ad599edfb..52a1bd40ffc 100644 --- a/unified/extractor/src/languages/mod.rs +++ b/unified/extractor/src/languages/mod.rs @@ -3,6 +3,15 @@ use codeql_extractor::extractor::simple; #[path = "swift/swift.rs"] mod swift; +/// swift-syntax JSON -> `yeast::Ast` adapter for the Swift front-end. +/// +/// Currently exercised by tests and the forthcoming runtime extraction path; +/// `allow(dead_code)` because this is a binary crate, so its public API isn't +/// counted as used until the binary itself calls it. +#[path = "swift/adapter.rs"] +#[allow(dead_code)] +pub mod swift_adapter; + /// Shared YEAST output AST schema for all languages. pub(crate) const OUTPUT_AST_SCHEMA: &str = include_str!("../../ast_types.yml"); diff --git a/unified/swift-syntax-rs/src/yeast_adapter.rs b/unified/extractor/src/languages/swift/adapter.rs similarity index 88% rename from unified/swift-syntax-rs/src/yeast_adapter.rs rename to unified/extractor/src/languages/swift/adapter.rs index e2adf1df5a6..37d5aac00d2 100644 --- a/unified/swift-syntax-rs/src/yeast_adapter.rs +++ b/unified/extractor/src/languages/swift/adapter.rs @@ -1,6 +1,10 @@ -//! Adapter that converts the swift-syntax JSON tree (see [`crate::parse_to_json`]) -//! into a [`yeast::Ast`], the in-memory format the CodeQL desugaring rules -//! operate on. +//! Converts the swift-syntax JSON syntax tree into a [`yeast::Ast`], the +//! in-memory format the CodeQL desugaring rules operate on. +//! +//! The JSON tree is produced by the `swift-syntax-rs` crate's Swift FFI shim +//! (`parse_to_json`). This module is pure Rust (only `yeast` + `serde_json`), +//! so the extractor consumes swift-syntax output without pulling in the Swift +//! toolchain (the JSON is produced out-of-process). //! //! The mapping mirrors tree-sitter's node model, which is what yeast (and the //! extractor's rewrite rules) expect: @@ -15,9 +19,8 @@ //! list-valued field maps directly to that field holding several children. //! //! Note: this preserves swift-syntax's own kind/field names. Aligning those -//! names with the tree-sitter-swift schema (so the existing rewrite rules fire) -//! is a separate, later step; this module is only concerned with getting the -//! tree into yeast's format. +//! names with the tree-sitter-swift schema (so the rewrite rules in +//! [`super::swift`] fire) is done incrementally in the rules. use std::collections::BTreeMap; @@ -409,40 +412,4 @@ mod tests { "comment should not appear as an AST node" ); } - - /// End-to-end: real Swift source parsed by the shim, then adapted into a - /// `yeast::Ast`. Requires the Swift toolchain (like the crate's FFI tests). - #[test] - fn end_to_end_from_swift_source() { - let json = crate::parse_to_json("func f(n: Int) -> Int { return n } // trailing") - .expect("parsing should succeed"); - let adapted = json_to_ast(&json).expect("adapter should succeed"); - let ast = &adapted.ast; - - let root = ast.get_node(ast.get_root()).expect("root exists"); - assert_eq!(root.kind_name(), "sourceFile"); - - // The tree contains a `functionDecl` layout node and an anonymous - // `func` keyword token keyed by its text. - let mut kinds: Vec<&str> = ast.nodes().iter().map(|n| n.kind_name()).collect(); - kinds.sort_unstable(); - assert!( - kinds.contains(&"functionDecl"), - "expected a functionDecl node, got kinds: {kinds:?}" - ); - assert!( - kinds.contains(&"func"), - "expected an anonymous `func` token, got kinds: {kinds:?}" - ); - - // The trailing comment is recovered into the side channel. - assert!( - adapted - .trivia - .iter() - .any(|t| t.kind == "lineComment" && t.text == "// trailing"), - "expected the trailing comment in the trivia side channel, got: {:?}", - adapted.trivia - ); - } } diff --git a/unified/extractor/src/languages/swift/swift.rs b/unified/extractor/src/languages/swift/swift.rs index 74809ea18e4..13f1a6beadf 100644 --- a/unified/extractor/src/languages/swift/swift.rs +++ b/unified/extractor/src/languages/swift/swift.rs @@ -135,6 +135,25 @@ fn translation_rules() -> Vec> { // Declarations may be wrapped in local/global wrapper nodes. rule!((global_declaration _ @inner) => stmt { inner }), rule!((local_declaration _ @inner) => stmt { inner }), + // ---- swift-syntax front-end (minimal hook-up) ---- + // These rules target the swift-syntax AST (camelCase kind names), + // produced by the sibling `adapter` module. They coexist with the + // tree-sitter rules (snake_case names): rules are dispatched by exact + // kind name, and the two name spaces never collide, so these are inert + // on the tree-sitter path. Only the minimal top-level mapping lives here + // to demonstrate the pipeline end-to-end; the full translation is added + // separately. Unmatched swift-syntax nodes fall through to the + // `unsupported_node` fallback at the end. + // + // `sourceFile` holds its top-level statements in an (elided) + // `statements` collection; each element is a `codeBlockItem` wrapping + // the real node. + rule!( + (sourceFile statements: _* @items) + => + (top_level body: (block stmt: {items})) + ), + rule!((codeBlockItem item: @item) => stmt { item }), // ---- Literals ---- rule!((integer_literal) => (int_literal)), rule!((hex_literal) => (int_literal)), diff --git a/unified/extractor/tests/fixtures/let_x.swiftsyntax.json b/unified/extractor/tests/fixtures/let_x.swiftsyntax.json new file mode 100644 index 00000000000..6c3d18e2fef --- /dev/null +++ b/unified/extractor/tests/fixtures/let_x.swiftsyntax.json @@ -0,0 +1,196 @@ +{ + "endOfFileToken": { + "kind": "token", + "range": { + "end": { + "column": 1, + "line": 2, + "offset": 10 + }, + "start": { + "column": 1, + "line": 2, + "offset": 10 + } + }, + "text": "", + "tokenKind": "endOfFile" + }, + "kind": "sourceFile", + "range": { + "end": { + "column": 1, + "line": 2, + "offset": 10 + }, + "start": { + "column": 1, + "line": 1, + "offset": 0 + } + }, + "statements": [ + { + "item": { + "attributes": [], + "bindings": [ + { + "initializer": { + "equal": { + "kind": "token", + "range": { + "end": { + "column": 8, + "line": 1, + "offset": 7 + }, + "start": { + "column": 7, + "line": 1, + "offset": 6 + } + }, + "text": "=", + "tokenKind": "equal" + }, + "kind": "initializerClause", + "range": { + "end": { + "column": 10, + "line": 1, + "offset": 9 + }, + "start": { + "column": 7, + "line": 1, + "offset": 6 + } + }, + "value": { + "kind": "integerLiteralExpr", + "literal": { + "kind": "token", + "range": { + "end": { + "column": 10, + "line": 1, + "offset": 9 + }, + "start": { + "column": 9, + "line": 1, + "offset": 8 + } + }, + "text": "1", + "tokenKind": "integerLiteral(\"1\")" + }, + "range": { + "end": { + "column": 10, + "line": 1, + "offset": 9 + }, + "start": { + "column": 9, + "line": 1, + "offset": 8 + } + } + } + }, + "kind": "patternBinding", + "pattern": { + "identifier": { + "kind": "token", + "range": { + "end": { + "column": 6, + "line": 1, + "offset": 5 + }, + "start": { + "column": 5, + "line": 1, + "offset": 4 + } + }, + "text": "x", + "tokenKind": "identifier(\"x\")" + }, + "kind": "identifierPattern", + "range": { + "end": { + "column": 6, + "line": 1, + "offset": 5 + }, + "start": { + "column": 5, + "line": 1, + "offset": 4 + } + } + }, + "range": { + "end": { + "column": 10, + "line": 1, + "offset": 9 + }, + "start": { + "column": 5, + "line": 1, + "offset": 4 + } + } + } + ], + "bindingSpecifier": { + "kind": "token", + "range": { + "end": { + "column": 4, + "line": 1, + "offset": 3 + }, + "start": { + "column": 1, + "line": 1, + "offset": 0 + } + }, + "text": "let", + "tokenKind": "keyword(SwiftSyntax.Keyword.let)" + }, + "kind": "variableDecl", + "modifiers": [], + "range": { + "end": { + "column": 10, + "line": 1, + "offset": 9 + }, + "start": { + "column": 1, + "line": 1, + "offset": 0 + } + } + }, + "kind": "codeBlockItem", + "range": { + "end": { + "column": 10, + "line": 1, + "offset": 9 + }, + "start": { + "column": 1, + "line": 1, + "offset": 0 + } + } + } + ] +} diff --git a/unified/extractor/tests/swift_syntax_pipeline.rs b/unified/extractor/tests/swift_syntax_pipeline.rs new file mode 100644 index 00000000000..cdaae1f47c6 --- /dev/null +++ b/unified/extractor/tests/swift_syntax_pipeline.rs @@ -0,0 +1,49 @@ +//! Integration test for the swift-syntax front-end pipeline: +//! +//! swift-syntax JSON -> `swift_adapter::json_to_ast` -> yeast `Ast` +//! -> `Desugarer::run_from_ast` (the real Swift translation rules) -> dump. +//! +//! This exercises the whole chain *without* the Swift toolchain: the JSON +//! fixture is a real `parse_to_json` dump (see the file header) fed through the +//! pure-Rust adapter module. It verifies the desugarer runs end-to-end over an +//! externally-built AST. + +use yeast::dump::dump_ast; + +#[path = "../src/languages/mod.rs"] +mod languages; + +/// A real `swift-syntax-rs` JSON dump of the Swift source `let x = 1`. +const LET_X_JSON: &str = include_str!("fixtures/let_x.swiftsyntax.json"); + +#[test] +fn swift_syntax_json_runs_through_the_desugarer() { + let lang = languages::all_language_specs() + .into_iter() + .find(|l| l.file_globs.iter().any(|g| g.contains("swift"))) + .expect("swift language spec"); + let desugarer = lang.desugar.as_deref().expect("swift desugarer"); + + // Adapt the swift-syntax JSON into a yeast AST (pure Rust, no Swift FFI). + let adapted = + languages::swift_adapter::json_to_ast(LET_X_JSON).expect("adapter should succeed"); + assert_eq!( + adapted + .ast + .get_node(adapted.ast.get_root()) + .unwrap() + .kind_name(), + "sourceFile" + ); + + // Run the real Swift desugaring rules over the externally-built AST. The + // top-level swift-syntax rules map `sourceFile` to `top_level`/`block`; + // kinds without swift-syntax rules yet fall back to `unsupported_node`. + let desugared = desugarer + .run_from_ast(adapted.ast) + .expect("desugaring an externally-built AST should not error"); + + let dump = dump_ast(&desugared, desugared.get_root(), ""); + assert!(dump.contains("top_level"), "unexpected dump: {dump}"); + assert!(dump.contains("block"), "unexpected dump: {dump}"); +} diff --git a/unified/swift-syntax-rs/BUILD.bazel b/unified/swift-syntax-rs/BUILD.bazel index 3a2e5ed0796..9db40d4db30 100644 --- a/unified/swift-syntax-rs/BUILD.bazel +++ b/unified/swift-syntax-rs/BUILD.bazel @@ -35,8 +35,6 @@ rust_library( edition = "2024", deps = [ ":swift_syntax_ffi", - "//shared/yeast", - "@vendor_ts__serde_json-1.0.145//:serde_json", ], ) diff --git a/unified/swift-syntax-rs/Cargo.toml b/unified/swift-syntax-rs/Cargo.toml index 57624646848..6957fb0e50e 100644 --- a/unified/swift-syntax-rs/Cargo.toml +++ b/unified/swift-syntax-rs/Cargo.toml @@ -12,7 +12,3 @@ path = "src/lib.rs" [[bin]] name = "swift-syntax-parse" path = "src/main.rs" - -[dependencies] -serde_json = "1.0" -yeast = { path = "../../shared/yeast" } diff --git a/unified/swift-syntax-rs/README.md b/unified/swift-syntax-rs/README.md index 08cabeb736b..a2d66495a54 100644 --- a/unified/swift-syntax-rs/README.md +++ b/unified/swift-syntax-rs/README.md @@ -171,26 +171,12 @@ echo 'let x = 1' | cargo run --bin swift-syntax-parse ## Converting to a yeast AST -For use in the CodeQL extractor, the JSON tree can be converted into a -[`yeast::Ast`](../../shared/yeast) — the in-memory format the extractor's -rewrite rules operate on — via [`yeast_adapter::json_to_ast`](src/yeast_adapter.rs): - -```rust -let json = swift_syntax_rs::parse_to_json("let x = 1")?; -let adapted = swift_syntax_rs::yeast_adapter::json_to_ast(&json)?; -let ast = adapted.ast; // the yeast::Ast -let comments = adapted.trivia; // side-channel comment/unexpectedText tokens -``` - -The adapter mirrors tree-sitter's node model, which is what yeast expects: -layout nodes and varying tokens (identifiers, literals, operators) become -**named** nodes; fixed tokens (keywords, punctuation) become **anonymous** -nodes keyed by their text. Comments (and `unexpectedText`) are harvested into a -side channel (`adapted.trivia`) during the same traversal rather than embedded -in the tree, matching how the extractor treats tree-sitter `extra` nodes. It -preserves swift-syntax's own kind/field names — aligning them with the -tree-sitter-swift schema so the existing rewrite rules fire is a separate, -later step. +The JSON tree is consumed by the CodeQL extractor, which converts it into a +[`yeast::Ast`](../../shared/yeast) — the in-memory format its rewrite rules +operate on. That adapter is a pure-Rust module living in the extractor +(`unified/extractor/src/languages/swift/adapter.rs`), so the extractor never +needs the Swift toolchain: it consumes the JSON produced out-of-process by this +crate's `parse_to_json` / the `swift-syntax-parse` binary. ## Layout @@ -198,5 +184,4 @@ later step. - `build.rs` — builds the Swift package and emits link/rpath flags (local `cargo` only). - `BUILD.bazel` — Bazel targets for the hermetic CI build (swift_library + rust targets). - `src/lib.rs` — safe Rust bindings (`parse_to_json`). -- `src/yeast_adapter.rs` — converts the JSON tree into a `yeast::Ast`. - `src/main.rs` — demo CLI. diff --git a/unified/swift-syntax-rs/src/lib.rs b/unified/swift-syntax-rs/src/lib.rs index aa711e322a4..2544c0bc408 100644 --- a/unified/swift-syntax-rs/src/lib.rs +++ b/unified/swift-syntax-rs/src/lib.rs @@ -3,9 +3,12 @@ //! //! The heavy lifting is done by a small Swift shim (see `swift/`) that links //! against `SwiftSyntax`/`SwiftParser` and exposes a tiny C ABI. This module -//! provides safe Rust bindings on top of that ABI. - -pub mod yeast_adapter; +//! provides safe Rust bindings on top of that ABI, exposing [`parse_to_json`] +//! which turns Swift source into a JSON syntax tree. +//! +//! Converting that JSON into a `yeast::Ast` (for the CodeQL extractor) is done +//! by the extractor's own pure-Rust adapter module, keeping the Swift toolchain +//! out of the extractor's build. use std::ffi::{CStr, CString}; use std::os::raw::c_char;