hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2026-07-23 04:02:07 +02:00
Code Issues Packages Projects Releases Wiki Activity

unified: Hook up swift-syntax AST

Browse Source 
At this point it's only a proof-of-concept translation -- it translates
`sourceFile` nodes, but everything else gets mapped to
'unsupported_node`.
This commit is contained in:
Taus
2026-07-07 15:56:56 +00:00
parent ddab6ccb35
commit 58ddeacde0
13 changed files with 404 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
Cargo.lock generated
View File

@@ -2853,10 +2853,6 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
[[package]]
name = "swift-syntax-rs"
version = "0.1.0"
dependencies = [
"serde_json",
"yeast",
]
[[package]]
name = "syn"

22
shared/yeast-schema/src/schema.rs
View File

@@ -167,6 +167,28 @@ impl Schema {
id
}
/// Register every kind (named and unnamed) and field *name* from `other`
/// into this schema (idempotent). Ids are assigned in this schema's own id
/// space; existing ids are unchanged.
///
/// This is used when running desugaring rules over an AST that was built
/// against a different schema (e.g. from an external parser): the rules
/// build output nodes whose kind/field names come from `other`, and those
/// names must resolve in the AST's own schema. Only names are needed — the
/// rule engine resolves kinds/fields by name and does not consult
/// `other`'s field-type or supertype information.
pub fn register_names_from(&mut self, other: &Schema) {
for name in other.kind_ids.keys() {
self.register_kind(name);
}
for name in other.unnamed_kind_ids.keys() {
self.register_unnamed_kind(name);
}
for name in other.field_ids.keys() {
self.register_field(name);
}
}
/// Track a name for a kind ID without registering it as named or
/// unnamed. Useful when importing tree-sitter ID tables that may
/// contain duplicate IDs across the named/unnamed split.

39
shared/yeast/src/lib.rs
View File

@@ -524,10 +524,15 @@ impl Ast {
self.schema.register_field(name)
}
fn union_source_range_of_children(
&self,
fields: &BTreeMap<FieldId, Vec<Id>>,
) -> Option<Range> {
/// Register every kind and field name from `schema` into this AST's schema
/// (idempotent). Used before desugaring an externally-built AST so that
/// rules can build output nodes whose kind/field names come from the
/// desugarer's output schema.
pub fn register_names_from_schema(&mut self, schema: &schema::Schema) {
self.schema.register_names_from(schema);
}
fn union_source_range_of_children(&self, fields: &BTreeMap<FieldId, Vec<Id>>) -> Option<Range> {
let mut start_byte: Option<usize> = None;
let mut end_byte: Option<usize> = None;
let mut start_point = Point { row: 0, column: 0 };
@@ -1448,6 +1453,18 @@ impl<'a, C: Clone + Default> Runner<'a, C> {
let mut user_ctx = C::default();
self.run_with_ctx(input, &mut user_ctx)
}
/// Run all phases over an already-built `ast`, using the default context
/// (`C::default()`). Unlike [`run_from_tree`](Self::run_from_tree), the AST
/// is supplied by the caller (e.g. built from an external parser's output)
/// rather than constructed from a tree-sitter tree. The caller is
/// responsible for ensuring the AST's schema knows any output kind/field
/// names the rules will build (see [`Ast::register_names_from_schema`]).
pub fn run_from_ast(&self, mut ast: Ast) -> Result<Ast, String> {
let mut user_ctx = C::default();
self.run_phases(&mut ast, &mut user_ctx)?;
Ok(ast)
}
}
// ---------------------------------------------------------------------------
@@ -1470,6 +1487,12 @@ pub trait Desugarer: Send + Sync {
/// Parse `tree` against `source` and run the desugaring pipeline.
/// Each call constructs a fresh default user context internally.
fn run_from_tree(&self, tree: &tree_sitter::Tree, source: &[u8]) -> Result<Ast, String>;
/// Run the desugaring pipeline over an already-built `ast` (e.g. produced
/// by an external parser rather than tree-sitter). The desugarer ensures
/// the AST's schema knows its output kind/field names before running the
/// rules. Each call constructs a fresh default user context internally.
fn run_from_ast(&self, ast: Ast) -> Result<Ast, String>;
}
/// A concrete [`Desugarer`] backed by a [`DesugaringConfig<C>`] for a
@@ -1507,4 +1530,12 @@ impl<C: Default + Clone + Send + Sync + 'static> Desugarer for ConcreteDesugarer
let runner = Runner::with_schema(self.language.clone(), &self.schema, &self.config.phases);
runner.run_from_tree(tree, source)
}
fn run_from_ast(&self, mut ast: Ast) -> Result<Ast, String> {
// The AST was built against its own (external) schema; make sure the
// output kind/field names the rules build are resolvable in it.
ast.register_names_from_schema(&self.schema);
let runner = Runner::with_schema(self.language.clone(), &self.schema, &self.config.phases);
runner.run_from_ast(ast)
}
}

53
shared/yeast/tests/test.rs
View File

@@ -266,6 +266,59 @@ fn test_query_match() {
assert!(captures.get_var("right").is_ok());
}
#[test]
fn test_run_from_ast_desugars_hand_built_tree() {
use std::collections::BTreeMap;
// Output schema for the desugared tree. Its kind/field names must become
// resolvable in the hand-built AST's schema for the rule to build them.
let schema_yaml = r#"
named:
assignment:
left: leaf
leaf:
"#;
// A rule over an *input* kind (`wrapper`) that is not in the output schema,
// rewriting to an output `assignment` node.
let rules: Vec<Rule> = vec![yeast::rule!(
(wrapper)
=>
(assignment left: (leaf "lit"))
)];
let lang: tree_sitter::Language = tree_sitter_ruby::LANGUAGE.into();
let config = DesugaringConfig::<()>::new()
.add_phase("test", PhaseKind::OneShot, rules)
.with_output_node_types_yaml(schema_yaml);
let desugarer = ConcreteDesugarer::new(lang, config).unwrap();
// Build the input AST by hand, as an external parser adapter would. The
// schema starts empty and gains the `wrapper` input kind on the fly.
let mut ast = Ast::with_schema(yeast::schema::Schema::new());
let wrapper_kind = ast.register_kind("wrapper");
let root = ast.create_node_with_range(
wrapper_kind,
NodeContent::DynamicString(String::new()),
BTreeMap::new(),
true,
None,
);
ast.set_root(root);
// Desugaring the hand-built AST applies the rule, producing `assignment`
// even though the AST was built against a schema with no output kinds.
let out = desugarer
.run_from_ast(ast)
.expect("run_from_ast should succeed");
let out_root = out.get_node(out.get_root()).expect("root exists");
assert_eq!(out_root.kind_name(), "assignment");
let dump = dump_ast(&out, out.get_root(), "");
assert!(dump.contains("assignment"), "unexpected dump: {dump}");
assert!(dump.contains("left"), "unexpected dump: {dump}");
assert!(dump.contains("leaf"), "unexpected dump: {dump}");
}
#[test]
fn test_query_no_match() {
let runner: Runner = Runner::new(tree_sitter_ruby::LANGUAGE.into(), &[]);

9
unified/extractor/src/languages/mod.rs
View File

@@ -3,6 +3,15 @@ use codeql_extractor::extractor::simple;
#[path = "swift/swift.rs"]
mod swift;
/// swift-syntax JSON -> `yeast::Ast` adapter for the Swift front-end.
///
/// Currently exercised by tests and the forthcoming runtime extraction path;
/// `allow(dead_code)` because this is a binary crate, so its public API isn't
/// counted as used until the binary itself calls it.
#[path = "swift/adapter.rs"]
#[allow(dead_code)]
pub mod swift_adapter;
/// Shared YEAST output AST schema for all languages.
pub(crate) const OUTPUT_AST_SCHEMA: &str = include_str!("../../ast_types.yml");

51
unified/swift-syntax-rs/src/yeast_adapter.rs → unified/extractor/src/languages/swift/adapter.rs
View File

@@ -1,6 +1,10 @@
//! Adapter that converts the swift-syntax JSON tree (see [`crate::parse_to_json`])
//! into a [`yeast::Ast`], the in-memory format the CodeQL desugaring rules
//! operate on.
//! Converts the swift-syntax JSON syntax tree into a [`yeast::Ast`], the
//! in-memory format the CodeQL desugaring rules operate on.
//!
//! The JSON tree is produced by the `swift-syntax-rs` crate's Swift FFI shim
//! (`parse_to_json`). This module is pure Rust (only `yeast` + `serde_json`),
//! so the extractor consumes swift-syntax output without pulling in the Swift
//! toolchain (the JSON is produced out-of-process).
//!
//! The mapping mirrors tree-sitter's node model, which is what yeast (and the
//! extractor's rewrite rules) expect:
@@ -15,9 +19,8 @@
//! list-valued field maps directly to that field holding several children.
//!
//! Note: this preserves swift-syntax's own kind/field names. Aligning those
//! names with the tree-sitter-swift schema (so the existing rewrite rules fire)
//! is a separate, later step; this module is only concerned with getting the
//! tree into yeast's format.
//! names with the tree-sitter-swift schema (so the rewrite rules in
//! [`super::swift`] fire) is done incrementally in the rules.
use std::collections::BTreeMap;
@@ -409,40 +412,4 @@ mod tests {
"comment should not appear as an AST node"
);
}
/// End-to-end: real Swift source parsed by the shim, then adapted into a
/// `yeast::Ast`. Requires the Swift toolchain (like the crate's FFI tests).
#[test]
fn end_to_end_from_swift_source() {
let json = crate::parse_to_json("func f(n: Int) -> Int { return n } // trailing")
.expect("parsing should succeed");
let adapted = json_to_ast(&json).expect("adapter should succeed");
let ast = &adapted.ast;
let root = ast.get_node(ast.get_root()).expect("root exists");
assert_eq!(root.kind_name(), "sourceFile");
// The tree contains a `functionDecl` layout node and an anonymous
// `func` keyword token keyed by its text.
let mut kinds: Vec<&str> = ast.nodes().iter().map(|n| n.kind_name()).collect();
kinds.sort_unstable();
assert!(
kinds.contains(&"functionDecl"),
"expected a functionDecl node, got kinds: {kinds:?}"
);
assert!(
kinds.contains(&"func"),
"expected an anonymous `func` token, got kinds: {kinds:?}"
);
// The trailing comment is recovered into the side channel.
assert!(
adapted
.trivia
.iter()
.any(|t| t.kind == "lineComment" && t.text == "// trailing"),
"expected the trailing comment in the trivia side channel, got: {:?}",
adapted.trivia
);
}
}

19
unified/extractor/src/languages/swift/swift.rs
View File

@@ -135,6 +135,25 @@ fn translation_rules() -> Vec<Rule<SwiftContext>> {
// Declarations may be wrapped in local/global wrapper nodes.
rule!((global_declaration _ @inner) => stmt { inner }),
rule!((local_declaration _ @inner) => stmt { inner }),
// ---- swift-syntax front-end (minimal hook-up) ----
// These rules target the swift-syntax AST (camelCase kind names),
// produced by the sibling `adapter` module. They coexist with the
// tree-sitter rules (snake_case names): rules are dispatched by exact
// kind name, and the two name spaces never collide, so these are inert
// on the tree-sitter path. Only the minimal top-level mapping lives here
// to demonstrate the pipeline end-to-end; the full translation is added
// separately. Unmatched swift-syntax nodes fall through to the
// `unsupported_node` fallback at the end.
//
// `sourceFile` holds its top-level statements in an (elided)
// `statements` collection; each element is a `codeBlockItem` wrapping
// the real node.
rule!(
(sourceFile statements: _* @items)
=>
(top_level body: (block stmt: {items}))
),
rule!((codeBlockItem item: @item) => stmt { item }),
// ---- Literals ----
rule!((integer_literal) => (int_literal)),
rule!((hex_literal) => (int_literal)),

196
unified/extractor/tests/fixtures/let_x.swiftsyntax.json vendored Normal file
View File

@@ -0,0 +1,196 @@
{
"endOfFileToken": {
"kind": "token",
"range": {
"end": {
"column": 1,
"line": 2,
"offset": 10
},
"start": {
"column": 1,
"line": 2,
"offset": 10
}
},
"text": "",
"tokenKind": "endOfFile"
},
"kind": "sourceFile",
"range": {
"end": {
"column": 1,
"line": 2,
"offset": 10
},
"start": {
"column": 1,
"line": 1,
"offset": 0
}
},
"statements": [
{
"item": {
"attributes": [],
"bindings": [
{
"initializer": {
"equal": {
"kind": "token",
"range": {
"end": {
"column": 8,
"line": 1,
"offset": 7
},
"start": {
"column": 7,
"line": 1,
"offset": 6
}
},
"text": "=",
"tokenKind": "equal"
},
"kind": "initializerClause",
"range": {
"end": {
"column": 10,
"line": 1,
"offset": 9
},
"start": {
"column": 7,
"line": 1,
"offset": 6
}
},
"value": {
"kind": "integerLiteralExpr",
"literal": {
"kind": "token",
"range": {
"end": {
"column": 10,
"line": 1,
"offset": 9
},
"start": {
"column": 9,
"line": 1,
"offset": 8
}
},
"text": "1",
"tokenKind": "integerLiteral(\"1\")"
},
"range": {
"end": {
"column": 10,
"line": 1,
"offset": 9
},
"start": {
"column": 9,
"line": 1,
"offset": 8
}
}
}
},
"kind": "patternBinding",
"pattern": {
"identifier": {
"kind": "token",
"range": {
"end": {
"column": 6,
"line": 1,
"offset": 5
},
"start": {
"column": 5,
"line": 1,
"offset": 4
}
},
"text": "x",
"tokenKind": "identifier(\"x\")"
},
"kind": "identifierPattern",
"range": {
"end": {
"column": 6,
"line": 1,
"offset": 5
},
"start": {
"column": 5,
"line": 1,
"offset": 4
}
}
},
"range": {
"end": {
"column": 10,
"line": 1,
"offset": 9
},
"start": {
"column": 5,
"line": 1,
"offset": 4
}
}
}
],
"bindingSpecifier": {
"kind": "token",
"range": {
"end": {
"column": 4,
"line": 1,
"offset": 3
},
"start": {
"column": 1,
"line": 1,
"offset": 0
}
},
"text": "let",
"tokenKind": "keyword(SwiftSyntax.Keyword.let)"
},
"kind": "variableDecl",
"modifiers": [],
"range": {
"end": {
"column": 10,
"line": 1,
"offset": 9
},
"start": {
"column": 1,
"line": 1,
"offset": 0
}
}
},
"kind": "codeBlockItem",
"range": {
"end": {
"column": 10,
"line": 1,
"offset": 9
},
"start": {
"column": 1,
"line": 1,
"offset": 0
}
}
}
]
}

49
unified/extractor/tests/swift_syntax_pipeline.rs Normal file
View File

@@ -0,0 +1,49 @@
//! Integration test for the swift-syntax front-end pipeline:
//!
//! swift-syntax JSON -> `swift_adapter::json_to_ast` -> yeast `Ast`
//! -> `Desugarer::run_from_ast` (the real Swift translation rules) -> dump.
//!
//! This exercises the whole chain *without* the Swift toolchain: the JSON
//! fixture is a real `parse_to_json` dump (see the file header) fed through the
//! pure-Rust adapter module. It verifies the desugarer runs end-to-end over an
//! externally-built AST.
use yeast::dump::dump_ast;
#[path = "../src/languages/mod.rs"]
mod languages;
/// A real `swift-syntax-rs` JSON dump of the Swift source `let x = 1`.
const LET_X_JSON: &str = include_str!("fixtures/let_x.swiftsyntax.json");
#[test]
fn swift_syntax_json_runs_through_the_desugarer() {
let lang = languages::all_language_specs()
.into_iter()
.find(|l| l.file_globs.iter().any(|g| g.contains("swift")))
.expect("swift language spec");
let desugarer = lang.desugar.as_deref().expect("swift desugarer");
// Adapt the swift-syntax JSON into a yeast AST (pure Rust, no Swift FFI).
let adapted =
languages::swift_adapter::json_to_ast(LET_X_JSON).expect("adapter should succeed");
assert_eq!(
adapted
.ast
.get_node(adapted.ast.get_root())
.unwrap()
.kind_name(),
"sourceFile"
);
// Run the real Swift desugaring rules over the externally-built AST. The
// top-level swift-syntax rules map `sourceFile` to `top_level`/`block`;
// kinds without swift-syntax rules yet fall back to `unsupported_node`.
let desugared = desugarer
.run_from_ast(adapted.ast)
.expect("desugaring an externally-built AST should not error");
let dump = dump_ast(&desugared, desugared.get_root(), "");
assert!(dump.contains("top_level"), "unexpected dump: {dump}");
assert!(dump.contains("block"), "unexpected dump: {dump}");
}

2
unified/swift-syntax-rs/BUILD.bazel
View File

@@ -35,8 +35,6 @@ rust_library(
edition = "2024",
deps = [
":swift_syntax_ffi",
"//shared/yeast",
"@vendor_ts__serde_json-1.0.145//:serde_json",
],
)

4
unified/swift-syntax-rs/Cargo.toml
View File

@@ -12,7 +12,3 @@ path = "src/lib.rs"
[[bin]]
name = "swift-syntax-parse"
path = "src/main.rs"
[dependencies]
serde_json = "1.0"
yeast = { path = "../../shared/yeast" }

27
unified/swift-syntax-rs/README.md
View File

@@ -171,26 +171,12 @@ echo 'let x = 1' | cargo run --bin swift-syntax-parse
## Converting to a yeast AST
For use in the CodeQL extractor, the JSON tree can be converted into a
[`yeast::Ast`](../../shared/yeast) — the in-memory format the extractor's
rewrite rules operate on — via [`yeast_adapter::json_to_ast`](src/yeast_adapter.rs):
```rust
let json = swift_syntax_rs::parse_to_json("let x = 1")?;
let adapted = swift_syntax_rs::yeast_adapter::json_to_ast(&json)?;
let ast = adapted.ast; // the yeast::Ast
let comments = adapted.trivia; // side-channel comment/unexpectedText tokens
```
The adapter mirrors tree-sitter's node model, which is what yeast expects:
layout nodes and varying tokens (identifiers, literals, operators) become
**named** nodes; fixed tokens (keywords, punctuation) become **anonymous**
nodes keyed by their text. Comments (and `unexpectedText`) are harvested into a
side channel (`adapted.trivia`) during the same traversal rather than embedded
in the tree, matching how the extractor treats tree-sitter `extra` nodes. It
preserves swift-syntax's own kind/field names — aligning them with the
tree-sitter-swift schema so the existing rewrite rules fire is a separate,
later step.
The JSON tree is consumed by the CodeQL extractor, which converts it into a
[`yeast::Ast`](../../shared/yeast) — the in-memory format its rewrite rules
operate on. That adapter is a pure-Rust module living in the extractor
(`unified/extractor/src/languages/swift/adapter.rs`), so the extractor never
needs the Swift toolchain: it consumes the JSON produced out-of-process by this
crate's `parse_to_json` / the `swift-syntax-parse` binary.
## Layout
@@ -198,5 +184,4 @@ later step.
- `build.rs` — builds the Swift package and emits link/rpath flags (local `cargo` only).
- `BUILD.bazel` — Bazel targets for the hermetic CI build (swift_library + rust targets).
- `src/lib.rs` — safe Rust bindings (`parse_to_json`).
- `src/yeast_adapter.rs` — converts the JSON tree into a `yeast::Ast`.
- `src/main.rs` — demo CLI.

9
unified/swift-syntax-rs/src/lib.rs
View File

@@ -3,9 +3,12 @@
//!
//! The heavy lifting is done by a small Swift shim (see `swift/`) that links
//! against `SwiftSyntax`/`SwiftParser` and exposes a tiny C ABI. This module
//! provides safe Rust bindings on top of that ABI.
pub mod yeast_adapter;
//! provides safe Rust bindings on top of that ABI, exposing [`parse_to_json`]
//! which turns Swift source into a JSON syntax tree.
//!
//! Converting that JSON into a `yeast::Ast` (for the CodeQL extractor) is done
//! by the extractor's own pure-Rust adapter module, keeping the Swift toolchain
//! out of the extractor's build.
use std::ffi::{CStr, CString};
use std::os::raw::c_char;