hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2026-07-28 22:36:50 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #22195 from github/tausbn/swift-syntax-rs

Browse Source 
unified: Add Swift parser based on `swift-syntax`
This commit is contained in:
Taus
2026-07-23 14:11:54 +02:00
committed by GitHub
parent 9836c1de57 38c9c84dcb
commit a39eab3e8c
30 changed files with 1929 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
Cargo.lock generated
View File

@@ -2850,6 +2850,10 @@ version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
[[package]]
name = "swift-syntax-rs"
version = "0.1.0"
[[package]]
name = "syn"
version = "2.0.106"

1
Cargo.toml
View File

@@ -10,6 +10,7 @@ members = [
"ruby/extractor",
"unified/extractor",
"unified/extractor/tree-sitter-swift",
"unified/swift-syntax-rs",
"rust/extractor",
"rust/extractor/macros",
"rust/ast-generator",

42
MODULE.bazel
View File

@@ -31,6 +31,13 @@ bazel_dep(name = "gazelle", version = "0.50.0")
bazel_dep(name = "rules_dotnet", version = "0.21.5-codeql.1")
bazel_dep(name = "googletest", version = "1.17.0.bcr.2")
bazel_dep(name = "rules_rust", version = "0.69.0")
bazel_dep(name = "rules_swift", version = "4.0.0-rc4")
bazel_dep(name = "swift-syntax", version = "603.0.2")
# Needed so we can `use_repo` `local_config_xcode` and
# `local_config_apple_cc_toolchains` below (referenced by the per-target
# Xcode-config transition in `unified/swift-syntax-rs/xcode_transition.bzl`).
bazel_dep(name = "apple_support", version = "2.6.1")
bazel_dep(name = "zstd", version = "1.5.7.bcr.1")
bazel_dep(name = "buildifier_prebuilt", version = "6.4.0", dev_dependency = True)
@@ -219,6 +226,41 @@ use_repo(
"swift-resource-dir-macos",
)
# Swift toolchain for building `unified/swift-syntax-rs`. On Linux we register
# a hermetic swift.org toolchain as the exec toolchain; on macOS `rules_swift`
# auto-registers `xcode_swift_toolchain` (host Xcode + OS-provided Swift
# runtime), which is not hermetic.
#
# The version is pinned as a literal rather than read from
# `unified/swift-syntax-rs/.swift-version` via `swift_version_file`: the latter
# makes the extension `module_ctx.read` a `//unified/...` label, which fails to
# resolve when this repo is consumed as a dependency module (`@@ql+`) whose
# `unified/swift-syntax-rs` package is not loadable in that context. Keep this
# in sync with `unified/swift-syntax-rs/.swift-version` (used by the `cargo`
# build) and the `swift-syntax` release in `swift/Package.swift`.
swift = use_extension("@rules_swift//swift:extensions.bzl", "swift")
swift.toolchain(
name = "swift_toolchain",
swift_version = "6.3.2",
)
use_repo(
swift,
"swift_toolchain",
"swift_toolchain_ubuntu22.04",
)
register_toolchains(
"@swift_toolchain//:swift_toolchain_exec_ubuntu22.04",
)
# `apple_support`'s xcode_config and CC toolchains, needed by the Xcode
# transition in `unified/swift-syntax-rs/xcode_transition.bzl`.
xcode_configure = use_extension("@apple_support//xcode:xcode_configure.bzl", "xcode_configure_extension")
use_repo(xcode_configure, "local_config_xcode")
apple_cc_configure = use_extension("@apple_support//crosstool:setup.bzl", "apple_cc_configure_extension")
use_repo(apple_cc_configure, "local_config_apple_cc_toolchains")
node = use_extension("@rules_nodejs//nodejs:extensions.bzl", "node")
node.toolchain(
name = "nodejs",

12
misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.bazel generated vendored
View File

@@ -553,6 +553,18 @@ alias(
tags = ["manual"],
)
alias(
name = "swift-syntax-rs-0.1.0",
actual = "@vendor_ts__swift-syntax-rs-0.1.0//:swift_syntax_rs",
tags = ["manual"],
)
alias(
name = "swift-syntax-rs",
actual = "@vendor_ts__swift-syntax-rs-0.1.0//:swift_syntax_rs",
tags = ["manual"],
)
alias(
name = "syn-2.0.106",
actual = "@vendor_ts__syn-2.0.106//:syn",

28
misc/bazel/3rdparty/tree_sitter_extractors_deps/defs.bzl generated vendored
View File

@@ -429,6 +429,10 @@ _NORMAL_DEPENDENCIES = {
"tree-sitter-language": Label("@vendor_ts__tree-sitter-language-0.1.5//:tree_sitter_language"),
},
},
"unified/swift-syntax-rs": {
_COMMON_CONDITION: {
},
},
}
_NORMAL_ALIASES = {
@@ -475,6 +479,10 @@ _NORMAL_ALIASES = {
_COMMON_CONDITION: {
},
},
"unified/swift-syntax-rs": {
_COMMON_CONDITION: {
},
},
}
_NORMAL_DEV_DEPENDENCIES = {
@@ -505,6 +513,8 @@ _NORMAL_DEV_DEPENDENCIES = {
},
"unified/extractor/tree-sitter-swift": {
},
"unified/swift-syntax-rs": {
},
}
_NORMAL_DEV_ALIASES = {
@@ -532,6 +542,8 @@ _NORMAL_DEV_ALIASES = {
},
"unified/extractor/tree-sitter-swift": {
},
"unified/swift-syntax-rs": {
},
}
_PROC_MACRO_DEPENDENCIES = {
@@ -557,6 +569,8 @@ _PROC_MACRO_DEPENDENCIES = {
},
"unified/extractor/tree-sitter-swift": {
},
"unified/swift-syntax-rs": {
},
}
_PROC_MACRO_ALIASES = {
@@ -582,6 +596,8 @@ _PROC_MACRO_ALIASES = {
},
"unified/extractor/tree-sitter-swift": {
},
"unified/swift-syntax-rs": {
},
}
_PROC_MACRO_DEV_DEPENDENCIES = {
@@ -607,6 +623,8 @@ _PROC_MACRO_DEV_DEPENDENCIES = {
},
"unified/extractor/tree-sitter-swift": {
},
"unified/swift-syntax-rs": {
},
}
_PROC_MACRO_DEV_ALIASES = {
@@ -634,6 +652,8 @@ _PROC_MACRO_DEV_ALIASES = {
},
"unified/extractor/tree-sitter-swift": {
},
"unified/swift-syntax-rs": {
},
}
_BUILD_DEPENDENCIES = {
@@ -663,6 +683,8 @@ _BUILD_DEPENDENCIES = {
"tree-sitter-generate": Label("@vendor_ts__tree-sitter-generate-0.26.8//:tree_sitter_generate"),
},
},
"unified/swift-syntax-rs": {
},
}
_BUILD_ALIASES = {
@@ -690,6 +712,8 @@ _BUILD_ALIASES = {
_COMMON_CONDITION: {
},
},
"unified/swift-syntax-rs": {
},
}
_BUILD_PROC_MACRO_DEPENDENCIES = {
@@ -715,6 +739,8 @@ _BUILD_PROC_MACRO_DEPENDENCIES = {
},
"unified/extractor/tree-sitter-swift": {
},
"unified/swift-syntax-rs": {
},
}
_BUILD_PROC_MACRO_ALIASES = {
@@ -740,6 +766,8 @@ _BUILD_PROC_MACRO_ALIASES = {
},
"unified/extractor/tree-sitter-swift": {
},
"unified/swift-syntax-rs": {
},
}
_CONDITIONS = {

12
shared/tree-sitter-extractor/src/extractor/mod.rs
View File

@@ -81,10 +81,18 @@ impl AstNode for yeast::Node {
yeast::Node::is_extra(self)
}
fn start_position(&self) -> tree_sitter::Point {
yeast::Node::start_position(self)
let p = yeast::Node::start_position(self);
tree_sitter::Point {
row: p.row,
column: p.column,
}
}
fn end_position(&self) -> tree_sitter::Point {
yeast::Node::end_position(self)
let p = yeast::Node::end_position(self);
tree_sitter::Point {
row: p.row,
column: p.column,
}
}
fn byte_range(&self) -> std::ops::Range<usize> {
yeast::Node::byte_range(self)

2
shared/yeast-macros/src/parse.rs
View File

@@ -903,7 +903,7 @@ pub fn parse_rule_top(input: TokenStream) -> Result<TokenStream> {
Ok(quote! {
{
let __query = #query_code;
yeast::Rule::new(__query, Box::new(|__ast: &mut yeast::Ast, mut __captures: yeast::captures::Captures, __fresh: &yeast::tree_builder::FreshScope, __source_range: Option<tree_sitter::Range>, __user_ctx: &mut _, __translator: yeast::TranslatorHandle<'_, _>| {
yeast::Rule::new(__query, Box::new(|__ast: &mut yeast::Ast, mut __captures: yeast::captures::Captures, __fresh: &yeast::tree_builder::FreshScope, __source_range: Option<yeast::Range>, __user_ctx: &mut _, __translator: yeast::TranslatorHandle<'_, _>| {
// Auto-translation prefix: recursively translate every
// captured node before invoking the user's transform body,
// except for `@@name` captures listed in `__skip` which the

22
shared/yeast-schema/src/schema.rs
View File

@@ -167,6 +167,28 @@ impl Schema {
id
}
/// Register every kind (named and unnamed) and field *name* from `other`
/// into this schema (idempotent). Ids are assigned in this schema's own id
/// space; existing ids are unchanged.
///
/// This is used when running desugaring rules over an AST that was built
/// against a different schema (e.g. from an external parser): the rules
/// build output nodes whose kind/field names come from `other`, and those
/// names must resolve in the AST's own schema. Only names are needed — the
/// rule engine resolves kinds/fields by name and does not consult
/// `other`'s field-type or supertype information.
pub fn register_names_from(&mut self, other: &Schema) {
for name in other.kind_ids.keys() {
self.register_kind(name);
}
for name in other.unnamed_kind_ids.keys() {
self.register_unnamed_kind(name);
}
for name in other.field_ids.keys() {
self.register_field(name);
}
}
/// Track a name for a kind ID without registering it as named or
/// unnamed. Useful when importing tree-sitter ID tables that may
/// contain duplicate IDs across the named/unnamed split.

10
shared/yeast/src/build.rs
View File

@@ -2,7 +2,7 @@ use std::collections::BTreeMap;
use crate::captures::Captures;
use crate::tree_builder::FreshScope;
use crate::{Ast, FieldId, Id, NodeContent, TranslatorHandle};
use crate::{Ast, FieldId, Id, NodeContent, Range, TranslatorHandle};
/// Context for building new AST nodes during a transformation.
///
@@ -34,7 +34,7 @@ pub struct BuildCtx<'a, C: 'a = ()> {
pub captures: &'a Captures,
pub fresh: &'a FreshScope,
/// Source range of the matched node, inherited by synthetic nodes.
pub source_range: Option<tree_sitter::Range>,
pub source_range: Option<Range>,
/// User-supplied context, accessible directly via `ctx.field` (via Deref).
pub user_ctx: &'a mut C,
/// Optional translator handle, populated when the context is built by
@@ -63,7 +63,7 @@ impl<'a, C> BuildCtx<'a, C> {
ast: &'a mut Ast,
captures: &'a Captures,
fresh: &'a FreshScope,
source_range: Option<tree_sitter::Range>,
source_range: Option<Range>,
user_ctx: &'a mut C,
) -> Self {
Self {
@@ -82,7 +82,7 @@ impl<'a, C> BuildCtx<'a, C> {
ast: &'a mut Ast,
captures: &'a Captures,
fresh: &'a FreshScope,
source_range: Option<tree_sitter::Range>,
source_range: Option<Range>,
user_ctx: &'a mut C,
translator: TranslatorHandle<'a, C>,
) -> Self {
@@ -143,7 +143,7 @@ impl<'a, C> BuildCtx<'a, C> {
&mut self,
kind: &'static str,
value: &str,
source_range: Option<tree_sitter::Range>,
source_range: Option<Range>,
) -> Id {
self.ast.create_named_token_with_range(
kind,

146
shared/yeast/src/lib.rs
View File

@@ -15,11 +15,32 @@ pub mod schema;
pub mod tree_builder;
mod visitor;
pub use range::{Point, Range};
pub use yeast_macros::{query, rule, rules, tree, trees};
use captures::Captures;
use query::QueryNode;
impl From<tree_sitter::Point> for Point {
fn from(p: tree_sitter::Point) -> Self {
Point {
row: p.row,
column: p.column,
}
}
}
impl From<tree_sitter::Range> for Range {
fn from(r: tree_sitter::Range) -> Self {
Range {
start_byte: r.start_byte,
end_byte: r.end_byte,
start_point: r.start_point.into(),
end_point: r.end_point.into(),
}
}
}
/// Node id: an index into the [`Ast`] arena. A newtype around `usize`
/// rather than a bare alias so that it can carry its own
/// [`YeastDisplay`] / [`YeastSourceRange`] / [`IntoFieldIds`] impls
@@ -106,7 +127,7 @@ pub trait YeastDisplay {
/// rule's source range. `Id` returns the referenced node's range, letting
/// `(kind #{capture})` carry the captured node's location.
pub trait YeastSourceRange {
fn yeast_source_range(&self, ast: &Ast) -> Option<tree_sitter::Range>;
fn yeast_source_range(&self, ast: &Ast) -> Option<Range>;
}
impl YeastDisplay for Id {
@@ -116,9 +137,9 @@ impl YeastDisplay for Id {
}
impl YeastSourceRange for Id {
fn yeast_source_range(&self, ast: &Ast) -> Option<tree_sitter::Range> {
fn yeast_source_range(&self, ast: &Ast) -> Option<Range> {
ast.get_node(*self).and_then(|n| match &n.content {
NodeContent::Range(r) => Some(r.clone()),
NodeContent::Range(r) => Some(*r),
_ => n.source_range,
})
}
@@ -134,7 +155,7 @@ macro_rules! impl_yeast_display_via_display {
}
impl YeastSourceRange for $t {
fn yeast_source_range(&self, _ast: &Ast) -> Option<tree_sitter::Range> {
fn yeast_source_range(&self, _ast: &Ast) -> Option<Range> {
None
}
}
@@ -157,7 +178,7 @@ impl<T: YeastDisplay + ?Sized> YeastDisplay for &T {
}
impl<T: YeastSourceRange + ?Sized> YeastSourceRange for &T {
fn yeast_source_range(&self, ast: &Ast) -> Option<tree_sitter::Range> {
fn yeast_source_range(&self, ast: &Ast) -> Option<Range> {
(**self).yeast_source_range(ast)
}
}
@@ -333,13 +354,41 @@ impl Ast {
ast
}
/// Construct an empty AST backed by `schema`, for building a tree
/// programmatically from a source other than a tree-sitter parse (e.g. an
/// external parser's output). Populate it with [`Ast::create_node`] /
/// [`Ast::create_node_with_range`] and then designate the root with
/// [`Ast::set_root`].
///
/// Node kind and field names may be registered in `schema` up front, or on
/// demand while building via [`Ast::register_kind`],
/// [`Ast::register_unnamed_kind`], and [`Ast::register_field`] (which return
/// the ids to pass to [`Ast::create_node_with_range`]). Passing a fresh
/// [`schema::Schema::new`] and registering names during construction is
/// therefore fine — see the swift-syntax adapter for an example.
pub fn with_schema(schema: schema::Schema) -> Self {
Self {
root: Id(0),
nodes: Vec::new(),
schema,
source: Vec::new(),
}
}
/// Set the original source bytes, used to resolve `NodeContent::Range`
/// nodes to text. Nodes built with inline `NodeContent::DynamicString`
/// content do not require this.
pub fn set_source(&mut self, source: Vec<u8>) {
self.source = source;
}
/// Returns the source text for `id`, resolving `NodeContent::Range`
/// against the stored source bytes when available.
pub fn source_text(&self, id: Id) -> String {
let Some(node) = self.get_node(id) else {
return String::new();
};
let read_range = |range: &tree_sitter::Range| {
let read_range = |range: &Range| {
let start = range.start_byte;
let end = range.end_byte;
if end <= self.source.len() && start <= end {
@@ -437,7 +486,7 @@ impl Ast {
content: NodeContent,
fields: BTreeMap<FieldId, Vec<Id>>,
is_named: bool,
source_range: Option<tree_sitter::Range>,
source_range: Option<Range>,
) -> Id {
let source_range = match &content {
// Parsed nodes already carry an exact source range in their content.
@@ -463,14 +512,37 @@ impl Ast {
Id(id)
}
fn union_source_range_of_children(
&self,
fields: &BTreeMap<FieldId, Vec<Id>>,
) -> Option<tree_sitter::Range> {
/// Register a named node kind, returning its id (idempotent). Lets callers
/// build an AST in a single pass, registering kinds as nodes are created
/// rather than pre-populating the schema.
pub fn register_kind(&mut self, name: &str) -> KindId {
self.schema.register_kind(name)
}
/// Register an anonymous (unnamed) token kind, returning its id
/// (idempotent). Anonymous tokens are keyed by their text (e.g. `"func"`).
pub fn register_unnamed_kind(&mut self, name: &str) -> KindId {
self.schema.register_unnamed_kind(name)
}
/// Register a field name, returning its id (idempotent).
pub fn register_field(&mut self, name: &str) -> FieldId {
self.schema.register_field(name)
}
/// Register every kind and field name from `schema` into this AST's schema
/// (idempotent). Used before desugaring an externally-built AST so that
/// rules can build output nodes whose kind/field names come from the
/// desugarer's output schema.
pub fn register_names_from_schema(&mut self, schema: &schema::Schema) {
self.schema.register_names_from(schema);
}
fn union_source_range_of_children(&self, fields: &BTreeMap<FieldId, Vec<Id>>) -> Option<Range> {
let mut start_byte: Option<usize> = None;
let mut end_byte: Option<usize> = None;
let mut start_point = tree_sitter::Point { row: 0, column: 0 };
let mut end_point = tree_sitter::Point { row: 0, column: 0 };
let mut start_point = Point { row: 0, column: 0 };
let mut end_point = Point { row: 0, column: 0 };
for child_ids in fields.values() {
for &child_id in child_ids {
@@ -513,7 +585,7 @@ impl Ast {
}
match (start_byte, end_byte) {
(Some(start_byte), Some(end_byte)) => Some(tree_sitter::Range {
(Some(start_byte), Some(end_byte)) => Some(Range {
start_byte,
end_byte,
start_point,
@@ -531,7 +603,7 @@ impl Ast {
&mut self,
kind: &'static str,
content: String,
source_range: Option<tree_sitter::Range>,
source_range: Option<Range>,
) -> Id {
let kind_id = self.schema.id_for_node_kind(kind).unwrap_or_else(|| {
panic!("create_named_token: node kind '{kind}' not found in schema")
@@ -604,7 +676,7 @@ impl Ast {
}
}
fn id_for_unnamed_node_kind(&self, kind: &str) -> Option<KindId> {
pub fn id_for_unnamed_node_kind(&self, kind: &str) -> Option<KindId> {
let id = self.schema.id_for_unnamed_node_kind(kind).unwrap_or(0);
if id == 0 {
None
@@ -624,7 +696,7 @@ pub struct Node {
/// For synthetic nodes, the source range of the original node they
/// were desugared from. Used for location information in TRAP output.
#[serde(skip)]
source_range: Option<tree_sitter::Range>,
source_range: Option<Range>,
is_named: bool,
is_missing: bool,
is_extra: bool,
@@ -652,11 +724,11 @@ impl Node {
self.is_error
}
fn fake_point(&self) -> tree_sitter::Point {
tree_sitter::Point { row: 0, column: 0 }
fn fake_point(&self) -> Point {
Point { row: 0, column: 0 }
}
pub fn start_position(&self) -> tree_sitter::Point {
pub fn start_position(&self) -> Point {
match self.content {
NodeContent::Range(range) => range.start_point,
_ => self
@@ -665,7 +737,7 @@ impl Node {
}
}
pub fn end_position(&self) -> tree_sitter::Point {
pub fn end_position(&self) -> Point {
match self.content {
NodeContent::Range(range) => range.end_point,
_ => self
@@ -714,7 +786,7 @@ impl Node {
/// or a new string if the node is synthesized.
#[derive(PartialEq, Eq, Debug, Clone, Serialize)]
pub enum NodeContent {
Range(#[serde(with = "range::Range")] tree_sitter::Range),
Range(Range),
String(&'static str),
DynamicString(String),
}
@@ -727,7 +799,7 @@ impl From<&'static str> for NodeContent {
impl From<tree_sitter::Range> for NodeContent {
fn from(value: tree_sitter::Range) -> Self {
NodeContent::Range(value)
NodeContent::Range(value.into())
}
}
@@ -854,7 +926,7 @@ pub type Transform<C = ()> = Box<
&mut Ast,
Captures,
&tree_builder::FreshScope,
Option<tree_sitter::Range>,
Option<Range>,
&mut C,
TranslatorHandle<'_, C>,
) -> Result<Vec<Id>, String>
@@ -1387,6 +1459,18 @@ impl<'a, C: Clone + Default> Runner<'a, C> {
let mut user_ctx = C::default();
self.run_with_ctx(input, &mut user_ctx)
}
/// Run all phases over an already-built `ast`, using the default context
/// (`C::default()`). Unlike [`run_from_tree`](Self::run_from_tree), the AST
/// is supplied by the caller (e.g. built from an external parser's output)
/// rather than constructed from a tree-sitter tree. The caller is
/// responsible for ensuring the AST's schema knows any output kind/field
/// names the rules will build (see [`Ast::register_names_from_schema`]).
pub fn run_from_ast(&self, mut ast: Ast) -> Result<Ast, String> {
let mut user_ctx = C::default();
self.run_phases(&mut ast, &mut user_ctx)?;
Ok(ast)
}
}
// ---------------------------------------------------------------------------
@@ -1409,6 +1493,12 @@ pub trait Desugarer: Send + Sync {
/// Parse `tree` against `source` and run the desugaring pipeline.
/// Each call constructs a fresh default user context internally.
fn run_from_tree(&self, tree: &tree_sitter::Tree, source: &[u8]) -> Result<Ast, String>;
/// Run the desugaring pipeline over an already-built `ast` (e.g. produced
/// by an external parser rather than tree-sitter). The desugarer ensures
/// the AST's schema knows its output kind/field names before running the
/// rules. Each call constructs a fresh default user context internally.
fn run_from_ast(&self, ast: Ast) -> Result<Ast, String>;
}
/// A concrete [`Desugarer`] backed by a [`DesugaringConfig<C>`] for a
@@ -1446,4 +1536,12 @@ impl<C: Default + Clone + Send + Sync + 'static> Desugarer for ConcreteDesugarer
let runner = Runner::with_schema(self.language.clone(), &self.schema, &self.config.phases);
runner.run_from_tree(tree, source)
}
fn run_from_ast(&self, mut ast: Ast) -> Result<Ast, String> {
// The AST was built against its own (external) schema; make sure the
// output kind/field names the rules build are resolvable in it.
ast.register_names_from_schema(&self.schema);
let runner = Runner::with_schema(self.language.clone(), &self.schema, &self.config.phases);
runner.run_from_ast(ast)
}
}

29
shared/yeast/src/range.rs
View File

@@ -1,21 +1,32 @@
//! (de)-serialize helpers for tree_sitter::Range
//! Location types for yeast ASTs.
//!
//! These are plain owned structs (mirroring tree-sitter's `Point`/`Range`) so
//! that code building an AST programmatically can populate node locations
//! without depending on tree-sitter. Conversions from the tree-sitter types
//! live in the crate root, next to the `from_tree` construction path.
use serde::{Deserialize, Serialize};
#[derive(Serialize, Deserialize)]
#[serde(remote = "tree_sitter::Point")]
/// A position in a source file: a 0-based `row` (line) and 0-based `column`
/// (UTF-8 byte offset within the line), matching tree-sitter's convention.
#[derive(Serialize, Deserialize, Clone, Copy, Debug, Default, PartialEq, Eq, Hash)]
pub struct Point {
pub row: usize,
pub column: usize,
}
#[derive(Serialize, Deserialize)]
#[serde(remote = "tree_sitter::Range")]
impl Point {
pub fn new(row: usize, column: usize) -> Self {
Self { row, column }
}
}
/// A half-open source range: byte offsets plus start/end [`Point`]s. The end is
/// exclusive, matching tree-sitter's convention.
#[derive(Serialize, Deserialize, Clone, Copy, Debug, Default, PartialEq, Eq, Hash)]
pub struct Range {
pub start_byte: usize,
pub end_byte: usize,
#[serde(with = "Point")]
pub start_point: tree_sitter::Point,
#[serde(with = "Point")]
pub end_point: tree_sitter::Point,
pub start_point: Point,
pub end_point: Point,
}

53
shared/yeast/tests/test.rs
View File

@@ -266,6 +266,59 @@ fn test_query_match() {
assert!(captures.get_var("right").is_ok());
}
#[test]
fn test_run_from_ast_desugars_hand_built_tree() {
use std::collections::BTreeMap;
// Output schema for the desugared tree. Its kind/field names must become
// resolvable in the hand-built AST's schema for the rule to build them.
let schema_yaml = r#"
named:
assignment:
left: leaf
leaf:
"#;
// A rule over an *input* kind (`wrapper`) that is not in the output schema,
// rewriting to an output `assignment` node.
let rules: Vec<Rule> = vec![yeast::rule!(
(wrapper)
=>
(assignment left: (leaf "lit"))
)];
let lang: tree_sitter::Language = tree_sitter_ruby::LANGUAGE.into();
let config = DesugaringConfig::<()>::new()
.add_phase("test", PhaseKind::OneShot, rules)
.with_output_node_types_yaml(schema_yaml);
let desugarer = ConcreteDesugarer::new(lang, config).unwrap();
// Build the input AST by hand, as an external parser adapter would. The
// schema starts empty and gains the `wrapper` input kind on the fly.
let mut ast = Ast::with_schema(yeast::schema::Schema::new());
let wrapper_kind = ast.register_kind("wrapper");
let root = ast.create_node_with_range(
wrapper_kind,
NodeContent::DynamicString(String::new()),
BTreeMap::new(),
true,
None,
);
ast.set_root(root);
// Desugaring the hand-built AST applies the rule, producing `assignment`
// even though the AST was built against a schema with no output kinds.
let out = desugarer
.run_from_ast(ast)
.expect("run_from_ast should succeed");
let out_root = out.get_node(out.get_root()).expect("root exists");
assert_eq!(out_root.kind_name(), "assignment");
let dump = dump_ast(&out, out.get_root(), "");
assert!(dump.contains("assignment"), "unexpected dump: {dump}");
assert!(dump.contains("left"), "unexpected dump: {dump}");
assert!(dump.contains("leaf"), "unexpected dump: {dump}");
}
#[test]
fn test_query_no_match() {
let runner: Runner = Runner::new(tree_sitter_ruby::LANGUAGE.into(), &[]);

9
unified/extractor/src/languages/mod.rs
View File

@@ -3,6 +3,15 @@ use codeql_extractor::extractor::simple;
#[path = "swift/swift.rs"]
mod swift;
/// swift-syntax JSON -> `yeast::Ast` adapter for the Swift front-end.
///
/// Currently exercised by tests and the forthcoming runtime extraction path;
/// `allow(dead_code)` because this is a binary crate, so its public API isn't
/// counted as used until the binary itself calls it.
#[path = "swift/adapter.rs"]
#[allow(dead_code)]
pub mod swift_adapter;
/// Shared YEAST output AST schema for all languages.
pub(crate) const OUTPUT_AST_SCHEMA: &str = include_str!("../../ast_types.yml");

415
unified/extractor/src/languages/swift/adapter.rs Normal file
View File

@@ -0,0 +1,415 @@
//! Converts the swift-syntax JSON syntax tree into a [`yeast::Ast`], the
//! in-memory format the CodeQL desugaring rules operate on.
//!
//! The JSON tree is produced by the `swift-syntax-rs` crate's Swift FFI shim
//! (`parse_to_json`). This module is pure Rust (only `yeast` + `serde_json`),
//! so the extractor consumes swift-syntax output without pulling in the Swift
//! toolchain (the JSON is produced out-of-process).
//!
//! The mapping mirrors tree-sitter's node model, which is what yeast (and the
//! extractor's rewrite rules) expect:
//!
//! * **Layout nodes** (e.g. `functionDecl`) and **varying tokens** (identifiers,
//! literals, operators — the ones whose text is not determined by their kind)
//! become **named** nodes, keyed by their kind name.
//! * **Fixed tokens** (keywords and punctuation, whose text is fully determined
//! by their kind) become **anonymous** nodes, keyed by their text — exactly
//! how tree-sitter models anonymous tokens (e.g. `"func"`, `"->"`).
//! * Collection nodes are already elided to JSON arrays upstream, so a
//! list-valued field maps directly to that field holding several children.
//!
//! Note: this preserves swift-syntax's own kind/field names. Aligning those
//! names with the tree-sitter-swift schema (so the rewrite rules in
//! [`super::swift`] fire) is done incrementally in the rules.
use std::collections::BTreeMap;
use serde_json::Value;
use yeast::schema::Schema;
use yeast::{Ast, Id, NodeContent, Point, Range};
/// A comment (or `unexpectedText`) recovered from the syntax tree's trivia.
///
/// These are collected into a side channel rather than embedded in the
/// [`yeast::Ast`], mirroring how the extractor treats tree-sitter `extra`
/// nodes: they carry a location and text but are not attached to a parent.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct TriviaToken {
/// The trivia kind (e.g. `lineComment`, `blockComment`, `docLineComment`,
/// `docBlockComment`, `unexpectedText`).
pub kind: String,
/// The verbatim source text of the piece (e.g. `// comment`).
pub text: String,
/// The source range the piece occupies.
pub range: Range,
}
/// The result of adapting a swift-syntax JSON tree: the [`yeast::Ast`] plus the
/// comment/`unexpectedText` trivia harvested from it (in source order).
pub struct AdaptedTree {
pub ast: Ast,
pub trivia: Vec<TriviaToken>,
}
/// swift-syntax `TokenKind` cases whose text is *not* determined by the kind
/// (i.e. `TokenKind.defaultText == nil`). These carry varying information and
/// are modelled as named leaf nodes; every other token is a fixed
/// keyword/punctuation token modelled as an anonymous token keyed by its text.
const VARYING_TOKEN_KINDS: &[&str] = &[
"identifier",
"integerLiteral",
"floatLiteral",
"stringSegment",
"binaryOperator",
"prefixOperator",
"postfixOperator",
"dollarIdentifier",
"regexLiteralPattern",
"rawStringPoundDelimiter",
"regexPoundDelimiter",
"shebang",
"unknown",
];
/// Keys of a node object that carry metadata rather than a structural child.
fn is_metadata_key(key: &str) -> bool {
matches!(
key,
"kind" | "range" | "tokenKind" | "text" | "leadingTrivia" | "trailingTrivia"
)
}
/// The classification of a JSON node into a yeast kind name and named-ness.
struct KindInfo {
/// The name under which the kind is registered in the schema.
name: String,
/// `true` for named nodes (layout nodes + varying tokens), `false` for
/// anonymous tokens (fixed keywords/punctuation).
is_named: bool,
/// The leaf text for tokens (empty for layout nodes).
text: String,
}
/// Determine the kind name / named-ness / text for a JSON node object.
fn classify(node: &Value) -> Result<KindInfo, String> {
let kind = node
.get("kind")
.and_then(Value::as_str)
.ok_or("node object is missing a string `kind`")?;
if kind != "token" {
// Layout node: named, keyed by its kind, no leaf text.
return Ok(KindInfo {
name: kind.to_string(),
is_named: true,
text: String::new(),
});
}
let token_kind = node
.get("tokenKind")
.and_then(Value::as_str)
.ok_or("token is missing a string `tokenKind`")?;
let text = node
.get("text")
.and_then(Value::as_str)
.unwrap_or("")
.to_string();
// The case name is the part before any `(payload)` in the debug rendering.
let case_name = token_kind.split('(').next().unwrap_or(token_kind);
if VARYING_TOKEN_KINDS.contains(&case_name) {
// Varying token: named leaf, keyed by its case name.
Ok(KindInfo {
name: case_name.to_string(),
is_named: true,
text,
})
} else {
// Fixed token: anonymous, keyed by its text (as tree-sitter does).
// `endOfFile` has empty text, so fall back to the case name there.
let name = if text.is_empty() {
case_name.to_string()
} else {
text.clone()
};
Ok(KindInfo {
name,
is_named: false,
text,
})
}
}
/// Iterate over a node object's structural (field, value) pairs in a stable
/// order, skipping metadata keys.
fn field_entries(node: &Value) -> Vec<(&str, &Value)> {
node.as_object()
.map(|map| {
map.iter()
.filter(|(k, _)| !is_metadata_key(k))
.map(|(k, v)| (k.as_str(), v))
.collect()
})
.unwrap_or_default()
}
/// The child node objects held by a field value, which is either a single node
/// object or an array of them (an elided collection).
fn children_of(value: &Value) -> Vec<&Value> {
match value {
Value::Array(items) => items.iter().collect(),
other => vec![other],
}
}
/// Recursively build `node` (and its descendants) into `ast`, returning its id.
///
/// This is a single traversal: each node's kind and field names are registered
/// in the schema on the fly, immediately before the node is created. Children
/// are built first so a parent's field lists reference existing ids. Any
/// comment/`unexpectedText` trivia carried by a token is harvested into
/// `trivia` during the same pass rather than embedded in the tree.
fn build(node: &Value, ast: &mut Ast, trivia: &mut Vec<TriviaToken>) -> Result<Id, String> {
let info = classify(node)?;
collect_trivia(node, trivia);
let mut fields: BTreeMap<u16, Vec<Id>> = BTreeMap::new();
for (field, value) in field_entries(node) {
let field_id = ast.register_field(field);
let mut ids = Vec::new();
for child in children_of(value) {
ids.push(build(child, ast, trivia)?);
}
fields.insert(field_id, ids);
}
let kind_id = if info.is_named {
ast.register_kind(&info.name)
} else {
ast.register_unnamed_kind(&info.name)
};
Ok(ast.create_node_with_range(
kind_id,
NodeContent::DynamicString(info.text),
fields,
info.is_named,
parse_range(node),
))
}
/// Harvest a token's `leadingTrivia`/`trailingTrivia` pieces (each already
/// filtered to comments/`unexpectedText` upstream) into `out`. Non-token nodes
/// have no trivia keys, so this is a no-op for them.
fn collect_trivia(node: &Value, out: &mut Vec<TriviaToken>) {
for key in ["leadingTrivia", "trailingTrivia"] {
let Some(Value::Array(pieces)) = node.get(key) else {
continue;
};
for piece in pieces {
let (Some(kind), Some(range)) = (
piece.get("kind").and_then(Value::as_str),
parse_range(piece),
) else {
continue;
};
let text = piece
.get("text")
.and_then(Value::as_str)
.unwrap_or("")
.to_string();
out.push(TriviaToken {
kind: kind.to_string(),
text,
range,
});
}
}
}
/// Parse a node's `range` into a [`yeast::Range`].
///
/// The JSON carries, for `start` and `end`, a 0-based UTF-8 file byte `offset`,
/// a 1-based `line`, and a 1-based UTF-8 byte `column`. yeast (like tree-sitter)
/// uses byte offsets with 0-based rows/columns and an exclusive end, so the
/// line/column are shifted down by one. swift-syntax's end position is already
/// exclusive, so the byte offsets map across directly.
fn parse_range(node: &Value) -> Option<Range> {
let range = node.get("range")?;
let point = |key: &str| -> Option<(usize, Point)> {
let p = range.get(key)?;
let offset = p.get("offset")?.as_u64()? as usize;
let line = p.get("line")?.as_u64()? as usize;
let column = p.get("column")?.as_u64()? as usize;
Some((
offset,
Point::new(line.saturating_sub(1), column.saturating_sub(1)),
))
};
let (start_byte, start_point) = point("start")?;
let (end_byte, end_point) = point("end")?;
Some(Range {
start_byte,
end_byte,
start_point,
end_point,
})
}
/// Convert a swift-syntax JSON tree (as produced by [`crate::parse_to_json`])
/// into a [`yeast::Ast`] plus the comment/`unexpectedText` trivia harvested
/// from it. Both are produced in a single traversal.
pub fn json_to_ast(json: &str) -> Result<AdaptedTree, String> {
let root: Value = serde_json::from_str(json).map_err(|e| format!("invalid JSON: {e}"))?;
let mut ast = Ast::with_schema(Schema::new());
let mut trivia = Vec::new();
let root_id = build(&root, &mut ast, &mut trivia)?;
ast.set_root(root_id);
// Emit trivia in source order (the traversal visits nodes bottom-up).
trivia.sort_by_key(|t| t.range.start_byte);
Ok(AdaptedTree { ast, trivia })
}
#[cfg(test)]
mod tests {
use super::*;
/// A hand-written JSON tree exercising layout nodes, a named (varying)
/// token, a fixed keyword token, and an elided collection field — so the
/// adapter is tested without needing the Swift toolchain.
fn sample_json() -> &'static str {
r#"{
"kind": "sourceFile",
"range": {"start":{"offset":0,"line":1,"column":1},"end":{"offset":9,"line":1,"column":10}},
"statements": [
{
"kind": "variableDecl",
"range": {"start":{"offset":0,"line":1,"column":1},"end":{"offset":9,"line":1,"column":10}},
"bindingSpecifier": {
"kind": "token",
"tokenKind": "keyword(SwiftSyntax.Keyword.let)",
"text": "let",
"range": {"start":{"offset":0,"line":1,"column":1},"end":{"offset":3,"line":1,"column":4}}
},
"name": {
"kind": "token",
"tokenKind": "identifier(\"x\")",
"text": "x",
"range": {"start":{"offset":4,"line":1,"column":5},"end":{"offset":5,"line":1,"column":6}}
}
}
]
}"#
}
#[test]
fn builds_ast_from_json() {
let ast = json_to_ast(sample_json())
.expect("adapter should succeed")
.ast;
let root = ast.get_root();
let root_node = ast.get_node(root).expect("root exists");
assert_eq!(root_node.kind_name(), "sourceFile");
assert!(root_node.is_named());
}
#[test]
fn classifies_named_and_anonymous_tokens() {
let ast = json_to_ast(sample_json())
.expect("adapter should succeed")
.ast;
// Walk all nodes and collect (kind_name, is_named) for the two tokens.
let mut let_named = None;
let mut ident_named = None;
for node in ast.nodes() {
match node.kind_name() {
"let" => let_named = Some(node.is_named()),
"identifier" => ident_named = Some(node.is_named()),
_ => {}
}
}
// Fixed keyword `let` is anonymous (keyed by its text "let").
assert_eq!(let_named, Some(false));
// Varying `identifier` is a named leaf.
assert_eq!(ident_named, Some(true));
}
#[test]
fn preserves_leaf_text() {
let ast = json_to_ast(sample_json())
.expect("adapter should succeed")
.ast;
let ident = ast
.nodes()
.iter()
.enumerate()
.find(|(_, n)| n.kind_name() == "identifier")
.map(|(i, _)| Id(i))
.expect("identifier node exists");
assert_eq!(ast.source_text(ident), "x");
}
#[test]
fn maps_source_locations() {
let ast = json_to_ast(sample_json())
.expect("adapter should succeed")
.ast;
let ident = ast
.nodes()
.iter()
.find(|n| n.kind_name() == "identifier")
.expect("identifier node exists");
// `x` is at file offset 4..5, line 1, column 5 (1-based) in the JSON,
// which maps to 0-based row 0, column 4 and byte range 4..5.
assert_eq!(ident.start_byte(), 4);
assert_eq!(ident.end_byte(), 5);
assert_eq!(ident.start_position(), Point::new(0, 4));
assert_eq!(ident.end_position(), Point::new(0, 5));
}
#[test]
fn collects_trivia_into_side_channel() {
// A token carrying a trailing line comment in its trivia.
let json = r#"{
"kind": "sourceFile",
"range": {"start":{"offset":0,"line":1,"column":1},"end":{"offset":14,"line":1,"column":15}},
"value": {
"kind": "token",
"tokenKind": "integerLiteral(\"1\")",
"text": "1",
"range": {"start":{"offset":0,"line":1,"column":1},"end":{"offset":1,"line":1,"column":2}},
"trailingTrivia": [
{
"kind": "lineComment",
"text": "// c",
"range": {"start":{"offset":2,"line":1,"column":3},"end":{"offset":6,"line":1,"column":7}}
}
]
}
}"#;
let adapted = json_to_ast(json).expect("adapter should succeed");
// The comment is in the side channel, with its text and location.
assert_eq!(adapted.trivia.len(), 1);
let comment = &adapted.trivia[0];
assert_eq!(comment.kind, "lineComment");
assert_eq!(comment.text, "// c");
assert_eq!(comment.range.start_byte, 2);
assert_eq!(comment.range.end_byte, 6);
// It is not embedded in the AST as a node.
assert!(
adapted
.ast
.nodes()
.iter()
.all(|n| n.kind_name() != "lineComment"),
"comment should not appear as an AST node"
);
}
}

19
unified/extractor/src/languages/swift/swift.rs
View File

@@ -135,6 +135,25 @@ fn translation_rules() -> Vec<Rule<SwiftContext>> {
// Declarations may be wrapped in local/global wrapper nodes.
rule!((global_declaration _ @inner) => stmt { inner }),
rule!((local_declaration _ @inner) => stmt { inner }),
// ---- swift-syntax front-end (minimal hook-up) ----
// These rules target the swift-syntax AST (camelCase kind names),
// produced by the sibling `adapter` module. They coexist with the
// tree-sitter rules (snake_case names): rules are dispatched by exact
// kind name, and the two name spaces never collide, so these are inert
// on the tree-sitter path. Only the minimal top-level mapping lives here
// to demonstrate the pipeline end-to-end; the full translation is added
// separately. Unmatched swift-syntax nodes fall through to the
// `unsupported_node` fallback at the end.
//
// `sourceFile` holds its top-level statements in an (elided)
// `statements` collection; each element is a `codeBlockItem` wrapping
// the real node.
rule!(
(sourceFile statements: _* @items)
=>
(top_level body: (block stmt: {items}))
),
rule!((codeBlockItem item: @item) => stmt { item }),
// ---- Literals ----
rule!((integer_literal) => (int_literal)),
rule!((hex_literal) => (int_literal)),

196
unified/extractor/tests/fixtures/let_x.swiftsyntax.json vendored Normal file
View File

@@ -0,0 +1,196 @@
{
"endOfFileToken": {
"kind": "token",
"range": {
"end": {
"column": 1,
"line": 2,
"offset": 10
},
"start": {
"column": 1,
"line": 2,
"offset": 10
}
},
"text": "",
"tokenKind": "endOfFile"
},
"kind": "sourceFile",
"range": {
"end": {
"column": 1,
"line": 2,
"offset": 10
},
"start": {
"column": 1,
"line": 1,
"offset": 0
}
},
"statements": [
{
"item": {
"attributes": [],
"bindings": [
{
"initializer": {
"equal": {
"kind": "token",
"range": {
"end": {
"column": 8,
"line": 1,
"offset": 7
},
"start": {
"column": 7,
"line": 1,
"offset": 6
}
},
"text": "=",
"tokenKind": "equal"
},
"kind": "initializerClause",
"range": {
"end": {
"column": 10,
"line": 1,
"offset": 9
},
"start": {
"column": 7,
"line": 1,
"offset": 6
}
},
"value": {
"kind": "integerLiteralExpr",
"literal": {
"kind": "token",
"range": {
"end": {
"column": 10,
"line": 1,
"offset": 9
},
"start": {
"column": 9,
"line": 1,
"offset": 8
}
},
"text": "1",
"tokenKind": "integerLiteral(\"1\")"
},
"range": {
"end": {
"column": 10,
"line": 1,
"offset": 9
},
"start": {
"column": 9,
"line": 1,
"offset": 8
}
}
}
},
"kind": "patternBinding",
"pattern": {
"identifier": {
"kind": "token",
"range": {
"end": {
"column": 6,
"line": 1,
"offset": 5
},
"start": {
"column": 5,
"line": 1,
"offset": 4
}
},
"text": "x",
"tokenKind": "identifier(\"x\")"
},
"kind": "identifierPattern",
"range": {
"end": {
"column": 6,
"line": 1,
"offset": 5
},
"start": {
"column": 5,
"line": 1,
"offset": 4
}
}
},
"range": {
"end": {
"column": 10,
"line": 1,
"offset": 9
},
"start": {
"column": 5,
"line": 1,
"offset": 4
}
}
}
],
"bindingSpecifier": {
"kind": "token",
"range": {
"end": {
"column": 4,
"line": 1,
"offset": 3
},
"start": {
"column": 1,
"line": 1,
"offset": 0
}
},
"text": "let",
"tokenKind": "keyword(SwiftSyntax.Keyword.let)"
},
"kind": "variableDecl",
"modifiers": [],
"range": {
"end": {
"column": 10,
"line": 1,
"offset": 9
},
"start": {
"column": 1,
"line": 1,
"offset": 0
}
}
},
"kind": "codeBlockItem",
"range": {
"end": {
"column": 10,
"line": 1,
"offset": 9
},
"start": {
"column": 1,
"line": 1,
"offset": 0
}
}
}
]
}

49
unified/extractor/tests/swift_syntax_pipeline.rs Normal file
View File

@@ -0,0 +1,49 @@
//! Integration test for the swift-syntax front-end pipeline:
//!
//! swift-syntax JSON -> `swift_adapter::json_to_ast` -> yeast `Ast`
//! -> `Desugarer::run_from_ast` (the real Swift translation rules) -> dump.
//!
//! This exercises the whole chain *without* the Swift toolchain: the JSON
//! fixture is a real `parse_to_json` dump (see the file header) fed through the
//! pure-Rust adapter module. It verifies the desugarer runs end-to-end over an
//! externally-built AST.
use yeast::dump::dump_ast;
#[path = "../src/languages/mod.rs"]
mod languages;
/// A real `swift-syntax-rs` JSON dump of the Swift source `let x = 1`.
const LET_X_JSON: &str = include_str!("fixtures/let_x.swiftsyntax.json");
#[test]
fn swift_syntax_json_runs_through_the_desugarer() {
let lang = languages::all_language_specs()
.into_iter()
.find(|l| l.file_globs.iter().any(|g| g.contains("swift")))
.expect("swift language spec");
let desugarer = lang.desugar.as_deref().expect("swift desugarer");
// Adapt the swift-syntax JSON into a yeast AST (pure Rust, no Swift FFI).
let adapted =
languages::swift_adapter::json_to_ast(LET_X_JSON).expect("adapter should succeed");
assert_eq!(
adapted
.ast
.get_node(adapted.ast.get_root())
.unwrap()
.kind_name(),
"sourceFile"
);
// Run the real Swift desugaring rules over the externally-built AST. The
// top-level swift-syntax rules map `sourceFile` to `top_level`/`block`;
// kinds without swift-syntax rules yet fall back to `unsupported_node`.
let desugared = desugarer
.run_from_ast(adapted.ast)
.expect("desugaring an externally-built AST should not error");
let dump = dump_ast(&desugared, desugared.get_root(), "");
assert!(dump.contains("top_level"), "unexpected dump: {dump}");
assert!(dump.contains("block"), "unexpected dump: {dump}");
}

2
unified/swift-syntax-rs/.gitignore vendored Normal file
View File

@@ -0,0 +1,2 @@
/target
/swift/.build

1
unified/swift-syntax-rs/.swift-version Normal file
View File

@@ -0,0 +1 @@
6.3.2

76
unified/swift-syntax-rs/BUILD.bazel Normal file
View File

@@ -0,0 +1,76 @@
load("@rules_rust//rust:defs.bzl", "rust_binary", "rust_library", "rust_test")
load(":swift_runtime.bzl", "swift_runtime_libs")
load(":xcode_transition.bzl", "xcode_transition_swift_library")
package(default_visibility = ["//visibility:public"])
swift_runtime_libs(
name = "swift_runtime_libs",
toolchain = "@swift_toolchain_ubuntu22.04//:files",
)
# Targets in this package require a Swift toolchain (Linux or macOS).
# `select()` gives us OR-of-OSes; other platforms get marked incompatible
# so `bazel build/test //...` skips them cleanly.
#
_SWIFT_SUPPORTED_PLATFORMS = select({
"@platforms//os:linux": [],
"@platforms//os:macos": [],
"//conditions:default": ["@platforms//:incompatible"],
})
# Swift FFI shim: wraps swift-syntax and exposes a small C ABI. The Rust
# targets below link against its `CcInfo`.
xcode_transition_swift_library(
name = "swift_syntax_ffi",
srcs = ["swift/Sources/SwiftSyntaxFFI/SwiftSyntaxFFI.swift"],
module_name = "SwiftSyntaxFFI",
target_compatible_with = _SWIFT_SUPPORTED_PLATFORMS,
deps = [
"@swift-syntax//:SwiftParser",
"@swift-syntax//:SwiftSyntax",
],
)
# Safe Rust bindings on top of the C ABI. Under Bazel the Swift side comes
# from `:swift_syntax_ffi`; `build.rs` is only used by the `cargo` workflow.
rust_library(
name = "swift_syntax_rs",
srcs = glob(
["src/**/*.rs"],
exclude = ["src/main.rs"],
),
edition = "2024",
target_compatible_with = _SWIFT_SUPPORTED_PLATFORMS,
deps = [
":swift_syntax_ffi",
],
)
rust_binary(
name = "swift-syntax-parse",
srcs = ["src/main.rs"],
# `rust_binary` doesn't copy the Swift runtime into runfiles the way
# `swift_binary` does. On Linux, ship the standalone toolchain's runtime;
# on macOS the OS provides it at `/usr/lib/swift` (rpath'd by
# `xcode_swift_toolchain`).
data = select({
"@platforms//os:macos": [],
"@platforms//os:linux": [":swift_runtime_libs"],
}),
edition = "2024",
target_compatible_with = _SWIFT_SUPPORTED_PLATFORMS,
deps = [":swift_syntax_rs"],
)
rust_test(
name = "swift_syntax_rs_test",
size = "small",
crate = ":swift_syntax_rs",
data = select({
"@platforms//os:macos": [],
"@platforms//os:linux": [":swift_runtime_libs"],
}),
edition = "2024",
target_compatible_with = _SWIFT_SUPPORTED_PLATFORMS,
)

14
unified/swift-syntax-rs/Cargo.toml Normal file
View File

@@ -0,0 +1,14 @@
[package]
name = "swift-syntax-rs"
description = "Rust wrapper around the swift-syntax package for parsing Swift source"
version = "0.1.0"
authors = ["GitHub"]
edition = "2024"
[lib]
name = "swift_syntax_rs"
path = "src/lib.rs"
[[bin]]
name = "swift-syntax-parse"
path = "src/main.rs"

203
unified/swift-syntax-rs/README.md Normal file
View File

@@ -0,0 +1,203 @@
# swift-syntax-rs
A Rust wrapper around the [swift-syntax](https://github.com/swiftlang/swift-syntax)
package, allowing Swift source code to be parsed from Rust.
Parsing is delegated to a small Swift shim (in [`swift/`](swift/)) that links
against `SwiftSyntax`/`SwiftParser` and exposes a tiny C ABI. The Rust crate
builds that shim (via `build.rs`) and provides safe bindings on top of it.
## Output format
The emitted JSON tree preserves the AST's named structure. Every node has a
`kind` and a `range` with `start`/`end` positions (UTF-8 `offset` plus 1-based
`line`/`column`). Beyond that:
- **Tokens** carry `text`, `tokenKind`, and — only when non-empty —
`leadingTrivia`/`trailingTrivia` arrays of `{ kind, text }` pieces.
- **Layout nodes** (e.g. `functionDecl`) embed their children directly as
members keyed by the child's name in the parent (`name`, `signature`,
`body`, …), alongside `kind`/`range`. Absent optional children are omitted.
- **Collection nodes** (e.g. `codeBlockItemList`) are elided: a list-valued
field is simply a JSON array of its elements (e.g. `parameters`, `statements`).
This drops the collection node's own `kind`/`range`.
Only meaningful trivia is kept — the four comment kinds (`lineComment`,
`blockComment`, `docLineComment`, `docBlockComment`) and `unexpectedText`
(source the parser skipped). Whitespace trivia is dropped, since node ranges
already encode positions.
### Example
Parsing `let x = 1 // c` produces the following (each `range` object is
abbreviated here as `…`):
```jsonc
{
"kind": "sourceFile",
"range": …,
"statements": [ // collection node elided to an array
{
"kind": "codeBlockItem",
"range": …,
"item": {
"kind": "variableDecl",
"range": …,
"attributes": [], // empty collection → empty array
"modifiers": [],
"bindingSpecifier": { // a token
"kind": "token",
"text": "let",
"tokenKind": "keyword(SwiftSyntax.Keyword.let)",
"range": …
},
"bindings": [
{
"kind": "patternBinding",
"range": …,
"pattern": {
"kind": "identifierPattern",
"range": …,
"identifier": { "kind": "token", "text": "x", "tokenKind": "identifier(\"x\")", "range": … }
},
"initializer": {
"kind": "initializerClause",
"range": …,
"equal": { "kind": "token", "text": "=", "tokenKind": "equal", "range": … },
"value": {
"kind": "integerLiteralExpr",
"range": …,
"literal": {
"kind": "token",
"text": "1",
"tokenKind": "integerLiteral(\"1\")",
"range": …,
"trailingTrivia": [ { "kind": "lineComment", "text": "// c" } ]
}
}
}
}
]
}
}
],
"endOfFileToken": { "kind": "token", "text": "", "tokenKind": "endOfFile", "range": … }
}
```
Note how `statements`, `bindings`, `attributes`, and `modifiers` are plain
arrays (their collection nodes are elided), layout children such as
`bindingSpecifier` and `initializer` are embedded by name, and the `// c`
comment rides along as `trailingTrivia` on the token it follows. Tokens without
trivia (most of them) simply omit the `leadingTrivia`/`trailingTrivia` keys.
## Prerequisites
The build does not depend on any particular version manager. You need:
- **Rust** — pinned to `1.88` by the repo-root [`rust-toolchain.toml`](../../rust-toolchain.toml),
which `rustup` picks up automatically.
- **Swift** — pinned to the version in [`.swift-version`](.swift-version)
(currently `6.3.2`), used to build `swift-syntax` `603.0.2`. Install it any way
you like — [swift.org](https://www.swift.org/install/) or
[swiftly](https://www.swift.org/swiftly/) (which reads `.swift-version`), or a
system package. Just make sure `swift` is on your `PATH` (or point `build.rs`
at it with the `SWIFT` environment variable).
On Debian/Ubuntu the Swift runtime also needs `libncurses6` (and related libs)
available on the system.
## Building & testing
With `cargo` and `swift` on `PATH`:
```sh
cargo build
cargo test
```
If your `swift`/`swiftc` are not on `PATH`, point the build at them explicitly:
```sh
SWIFT=/path/to/swift SWIFTC=/path/to/swiftc cargo build
```
The first build compiles `swift-syntax` and can take several minutes.
## Building with Bazel (CI)
CI builds this crate hermetically with Bazel. A Swift toolchain is downloaded
from swift.org by the official `rules_swift` standalone toolchain extension
(wired up in the repo-root `MODULE.bazel`), `swift-syntax` is pulled from the
Bazel Central Registry, and the FFI shim is compiled as a `swift_library` that
the Rust targets link against. `build.rs` is not used under Bazel; it only
builds the Swift shim for the local `cargo` workflow.
```sh
bazel build //unified/swift-syntax-rs:swift-syntax-parse
bazel test //unified/swift-syntax-rs:swift_syntax_rs_test
bazel run //unified/swift-syntax-rs:swift-syntax-parse < some.swift
```
Requirements:
- **`clang`** must be installed on the runner. `rules_swift` requires the Bazel
CC toolchain to use clang; the repo's `.bazelrc` already sets
`--repo_env=CC=clang`, so no extra flags are needed.
- The registered Swift toolchains cover **ubuntu24.04 / x86_64** and
**macOS / `xcode`** (Apple Silicon and Intel). Bazel selects the toolchain
matching the host. Targets are marked `target_compatible_with` these two
OSes, so on Windows Bazel skips them cleanly.
- **macOS only:** the Swift toolchain comes from the host Xcode installation
(`rules_swift` auto-registers `xcode_swift_toolchain`), which also needs
Xcode's CC toolchain and xcode_config; these are applied to the Swift
target via an incoming-edge Starlark transition (see
[`xcode_transition.bzl`](xcode_transition.bzl)), so other targets on macOS
keep using Bazel's default CC toolchain.
The Swift compiler version is kept in sync across three places: the
[`.swift-version`](.swift-version) file (read by the local `cargo`/`swift build`
and by [swiftly](https://www.swift.org/swiftly/)), the literal `swift_version`
pinned on `swift.toolchain(...)` in the root `MODULE.bazel` (the hermetic
swift.org **Linux** Bazel toolchain), and the `swift-syntax` release in
`swift/Package.swift`. On **macOS** the version is *not* pinned by the Bazel
build: `rules_swift` auto-registers the host `xcode_swift_toolchain`, which uses
whichever Swift ships with the installed Xcode. So the pin governs Linux (and
local) builds, while the macOS compiler version depends on the host Xcode.
(The Bazel toolchain pins a literal rather than reading `.swift-version` via
`swift_version_file`, because the latter makes the module extension read a
`//unified/...` label, which fails when this repo is consumed as a dependency
module.)
## Usage
Library:
```rust
let json = swift_syntax_rs::parse_to_json("let x = 1")?;
println!("{json}");
```
CLI (reads a file argument or stdin, prints the syntax tree as JSON):
```sh
echo 'let x = 1' | cargo run --bin swift-syntax-parse
```
## Converting to a yeast AST
The JSON tree is consumed by the CodeQL extractor, which converts it into a
[`yeast::Ast`](../../shared/yeast) — the in-memory format its rewrite rules
operate on. That adapter is a pure-Rust module living in the extractor
(`unified/extractor/src/languages/swift/adapter.rs`), so the extractor never
needs the Swift toolchain: it consumes the JSON produced out-of-process by this
crate's `parse_to_json` / the `swift-syntax-parse` binary.
## Layout
- `swift/` — Swift package exposing the `ssr_parse_json` / `ssr_string_free` C ABI.
- `build.rs` — builds the Swift package and emits link/rpath flags (local `cargo` only).
- `BUILD.bazel` — Bazel targets for the hermetic CI build (swift_library + rust targets).
- `src/lib.rs` — safe Rust bindings (`parse_to_json`).
- `src/main.rs` — demo CLI.

106
unified/swift-syntax-rs/build.rs Normal file
View File

@@ -0,0 +1,106 @@
use std::env;
use std::path::PathBuf;
use std::process::Command;
fn main() {
let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap());
let swift_dir = manifest_dir.join("swift");
// Emitting any `rerun-if-changed` disables Cargo's default whole-package
// scan, so we must list every input to the `swift build` below. Watch the
// package manifests, the pinned dependency lockfile, the pinned compiler
// version, and the entire Swift sources tree (a directory is scanned
// recursively). `.build/` is intentionally *not* watched (it is this
// build's output; watching it would cause perpetual rebuilds).
for input in [
swift_dir.join("Package.swift"),
swift_dir.join("Package.resolved"),
swift_dir.join("Sources"),
manifest_dir.join(".swift-version"),
] {
println!("cargo:rerun-if-changed={}", input.display());
}
println!("cargo:rerun-if-env-changed=SWIFT");
println!("cargo:rerun-if-env-changed=SWIFTC");
// Build the Swift FFI package as a release dynamic library.
let mut command = Command::new(swift_bin());
command
.args(["build", "-c", "release"])
.current_dir(&swift_dir);
apply_bare_repository_workaround(&mut command);
let status = command.status().unwrap_or_else(|e| {
panic!(
"failed to run `{swift} build`: {e}\n\
Install a Swift toolchain (see https://www.swift.org/install/, e.g. via \
swiftly) and ensure `swift` is on PATH, or set the `SWIFT` environment \
variable to the `swift` executable. The pinned version is in `.swift-version`.",
swift = swift_bin(),
)
});
assert!(status.success(), "`swift build` failed");
// Link against the freshly built dynamic library.
let build_dir = swift_dir.join(".build/release");
println!("cargo:rustc-link-search=native={}", build_dir.display());
println!("cargo:rustc-link-lib=dylib=SwiftSyntaxFFI");
println!("cargo:rustc-link-arg=-Wl,-rpath,{}", build_dir.display());
// The executable also needs to find the Swift runtime libraries at run time.
if let Some(runtime) = swift_runtime_dir() {
println!("cargo:rustc-link-search=native={}", runtime.display());
println!("cargo:rustc-link-arg=-Wl,-rpath,{}", runtime.display());
}
}
/// Query the active Swift toolchain for the directory containing its runtime
/// shared libraries (e.g. `libswiftCore.so`).
fn swift_runtime_dir() -> Option<PathBuf> {
let output = Command::new(swiftc_bin())
.arg("-print-target-info")
.output()
.ok()?;
if !output.status.success() {
return None;
}
let info = String::from_utf8_lossy(&output.stdout);
// Extract the value of `"runtimeResourcePath": "..."` without pulling in a
// JSON dependency.
let key = "\"runtimeResourcePath\"";
let start = info.find(key)?;
let rest = &info[start + key.len()..];
let colon = rest.find(':')?;
let after = &rest[colon + 1..];
let open = after.find('"')?;
let value_start = &after[open + 1..];
let close = value_start.find('"')?;
let resource_path = &value_start[..close];
Some(PathBuf::from(resource_path).join(if cfg!(target_os = "macos") { "macosx" } else { "linux" }))
}
/// The `swift` driver to invoke: `$SWIFT` if set, otherwise `swift` from `PATH`.
/// This keeps the build tool-agnostic — any Swift install works; no particular
/// version manager is required.
fn swift_bin() -> String {
env::var("SWIFT").unwrap_or_else(|_| "swift".to_string())
}
/// The `swiftc` compiler to invoke: `$SWIFTC` if set, otherwise `swiftc` from
/// `PATH`.
fn swiftc_bin() -> String {
env::var("SWIFTC").unwrap_or_else(|_| "swiftc".to_string())
}
/// Some environments (notably GitHub Codespaces) inject
/// `GIT_CONFIG_KEY_0=safe.bareRepository` / `GIT_CONFIG_VALUE_0=explicit`, which
/// breaks the cached bare git repositories `swift build` uses. When exactly that
/// key is present, relax it to `all` for the `swift build` subprocess only
/// (rather than unconditionally, which could clobber an unrelated
/// `GIT_CONFIG_VALUE_0`).
fn apply_bare_repository_workaround(command: &mut Command) {
if env::var("GIT_CONFIG_KEY_0").as_deref() == Ok("safe.bareRepository") {
command.env("GIT_CONFIG_VALUE_0", "all");
}
}

135
unified/swift-syntax-rs/src/lib.rs Normal file
View File

@@ -0,0 +1,135 @@
//! Rust wrapper around the [swift-syntax](https://github.com/swiftlang/swift-syntax)
//! package, allowing Swift source code to be parsed from Rust.
//!
//! The heavy lifting is done by a small Swift shim (see `swift/`) that links
//! against `SwiftSyntax`/`SwiftParser` and exposes a tiny C ABI. This module
//! provides safe Rust bindings on top of that ABI, exposing [`parse_to_json`]
//! which turns Swift source into a JSON syntax tree.
//!
//! Converting that JSON into a `yeast::Ast` (for the CodeQL extractor) is done
//! by the extractor's own pure-Rust adapter module, keeping the Swift toolchain
//! out of the extractor's build.
use std::ffi::{CStr, CString};
use std::os::raw::c_char;
// C ABI exported by the `SwiftSyntaxFFI` dynamic library.
unsafe extern "C" {
/// Parse a NUL-terminated Swift source string, returning a heap-allocated
/// NUL-terminated JSON string (or null on failure). The caller owns the
/// returned pointer and must release it with `ssr_string_free`.
fn ssr_parse_json(source: *const c_char) -> *mut c_char;
/// Free a string previously returned by `ssr_parse_json`.
fn ssr_string_free(ptr: *mut c_char);
}
/// Errors that can occur while parsing Swift source.
#[derive(Debug)]
pub enum ParseError {
/// The provided source contained an interior NUL byte.
NulByte,
/// The Swift shim returned no result. `SwiftParser` recovers from invalid
/// syntax (it always produces a tree, possibly with error nodes), so this
/// does *not* indicate a syntax error in the source — it means the shim
/// failed to serialize the tree to JSON or to allocate the result string.
SwiftFailure,
}
impl std::fmt::Display for ParseError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
ParseError::NulByte => write!(f, "source contained an interior NUL byte"),
ParseError::SwiftFailure => {
write!(f, "the swift-syntax shim failed to produce a JSON result")
}
}
}
}
impl std::error::Error for ParseError {}
/// Parse the given Swift `source` and return a JSON representation of its
/// syntax tree.
///
/// # Example
/// ```no_run
/// let json = swift_syntax_rs::parse_to_json("let x = 1").unwrap();
/// println!("{json}");
/// ```
pub fn parse_to_json(source: &str) -> Result<String, ParseError> {
let c_source = CString::new(source).map_err(|_| ParseError::NulByte)?;
// SAFETY: `c_source` is a valid NUL-terminated string for the duration of
// the call. The returned pointer, if non-null, is owned by us and freed via
// `ssr_string_free` before returning.
unsafe {
let ptr = ssr_parse_json(c_source.as_ptr());
if ptr.is_null() {
return Err(ParseError::SwiftFailure);
}
let json = CStr::from_ptr(ptr).to_string_lossy().into_owned();
ssr_string_free(ptr);
Ok(json)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn parses_simple_declaration() {
let json = parse_to_json("let x = 1").expect("parsing should succeed");
assert!(
json.contains("\"kind\":\"sourceFile\""),
"unexpected tree: {json}"
);
assert!(json.contains("\"text\":\"x\""), "unexpected tree: {json}");
// Source ranges are emitted for every node.
assert!(json.contains("\"range\""), "missing ranges: {json}");
assert!(
json.contains("\"line\"") && json.contains("\"column\"") && json.contains("\"offset\""),
"missing location fields: {json}"
);
}
#[test]
fn preserves_named_structure() {
let json = parse_to_json("let x = 1").expect("parsing should succeed");
// Layout children are embedded directly under their field name.
assert!(
json.contains("\"initializer\""),
"missing initializer field: {json}"
);
// Collection-valued fields are elided to plain JSON arrays.
assert!(
json.contains("\"statements\":["),
"statements should be an array: {json}"
);
assert!(
json.contains("\"bindings\":["),
"bindings should be an array: {json}"
);
}
#[test]
fn parses_empty_source() {
let json = parse_to_json("").expect("parsing empty source should succeed");
assert!(
json.contains("\"kind\":\"sourceFile\""),
"unexpected tree: {json}"
);
}
#[test]
fn captures_trivia() {
// A leading comment is kept as trivia on the token it precedes.
let json = parse_to_json("// hi\nlet x = 1").expect("parsing should succeed");
assert!(json.contains("\"leadingTrivia\""), "missing trivia: {json}");
assert!(
json.contains("lineComment"),
"comment trivia not captured: {json}"
);
}
}

42
unified/swift-syntax-rs/src/main.rs Normal file
View File

@@ -0,0 +1,42 @@
//! Small demo CLI: parse Swift source and print the syntax tree as JSON.
//!
//! Usage:
//! swift-syntax-parse [FILE]
//!
//! Reads from FILE if given, otherwise from standard input.
use std::io::Read;
use std::process::ExitCode;
fn main() -> ExitCode {
let source = match read_source() {
Ok(s) => s,
Err(e) => {
eprintln!("error reading source: {e}");
return ExitCode::FAILURE;
}
};
match swift_syntax_rs::parse_to_json(&source) {
Ok(json) => {
println!("{json}");
ExitCode::SUCCESS
}
Err(e) => {
eprintln!("error: {e}");
ExitCode::FAILURE
}
}
}
fn read_source() -> std::io::Result<String> {
let mut args = std::env::args().skip(1);
match args.next() {
Some(path) => std::fs::read_to_string(path),
None => {
let mut buf = String::new();
std::io::stdin().read_to_string(&mut buf)?;
Ok(buf)
}
}
}

15
unified/swift-syntax-rs/swift/Package.resolved Normal file
View File

@@ -0,0 +1,15 @@
{
"originHash" : "169957f8fbd882866f8f9f1c68cde813dca3560577939a8ffec366e3d77548e4",
"pins" : [
{
"identity" : "swift-syntax",
"kind" : "remoteSourceControl",
"location" : "https://github.com/swiftlang/swift-syntax.git",
"state" : {
"revision" : "79e4b74a295b6eb74a8b585e3a39d29e70c1dbd1",
"version" : "603.0.2"
}
}
],
"version" : 3
}

35
unified/swift-syntax-rs/swift/Package.swift Normal file
View File

@@ -0,0 +1,35 @@
// swift-tools-version:6.0
import PackageDescription
let package = Package(
name: "SwiftSyntaxFFI",
platforms: [
// swift-syntax 603 requires macOS 10.15; declare it explicitly
// rather than relying on the swift-tools-version default (10.13).
.macOS(.v10_15),
],
products: [
// Dynamic library so the produced .so records its dependency on the
// Swift runtime libraries, keeping the Rust link step simple.
.library(
name: "SwiftSyntaxFFI",
type: .dynamic,
targets: ["SwiftSyntaxFFI"]
)
],
dependencies: [
.package(
url: "https://github.com/swiftlang/swift-syntax.git",
exact: "603.0.2"
)
],
targets: [
.target(
name: "SwiftSyntaxFFI",
dependencies: [
.product(name: "SwiftSyntax", package: "swift-syntax"),
.product(name: "SwiftParser", package: "swift-syntax"),
]
)
]
)

181
unified/swift-syntax-rs/swift/Sources/SwiftSyntaxFFI/SwiftSyntaxFFI.swift Normal file
View File

@@ -0,0 +1,181 @@
import Foundation
import SwiftParser
// `@_spi(RawSyntax)` exposes the `childName(_:)` helper that maps a child's
// key path to its field name in the parent layout; used to emit named fields.
@_spi(RawSyntax) import SwiftSyntax
#if canImport(Glibc)
import Glibc
#elseif canImport(Darwin)
import Darwin
#endif
/// Convert an absolute position into an `{ offset, line, column }` dictionary.
///
/// `offset` is a UTF-8 byte offset; `line`/`column` are 1-based.
private func location(
_ position: AbsolutePosition,
_ converter: SourceLocationConverter
) -> [String: Any] {
let loc = converter.location(for: position)
return [
"offset": position.utf8Offset,
"line": loc.line,
"column": loc.column,
]
}
/// Trivia kinds worth preserving in the serialized tree. Comments carry
/// developer intent (including doc comments), and `unexpectedText` flags source
/// the parser had to skip. Whitespace and multi-line-string escape markers are
/// dropped: node ranges already encode positions, so they would only bloat the
/// output.
private let keptTriviaKinds: Set<String> = [
"lineComment",
"blockComment",
"docLineComment",
"docBlockComment",
"unexpectedText",
]
/// Serialize a trivia collection into an array of `{ kind, text, range }`
/// pieces, keeping only the kinds in `keptTriviaKinds`.
///
/// `start` is the absolute position of the first piece (a token's leading
/// trivia starts at `token.position`; its trailing trivia at
/// `token.endPositionBeforeTrailingTrivia`). Each piece's range is derived by
/// accumulating piece lengths, so kept pieces carry an exact source location.
private func serializeTrivia(
_ trivia: Trivia,
startingAt start: AbsolutePosition,
_ converter: SourceLocationConverter
) -> [Any] {
var result: [Any] = []
var offset = start.utf8Offset
for piece in trivia.pieces {
let length = piece.sourceLength.utf8Length
// The label of an enum case mirror is the case name (e.g. "spaces",
// "lineComment"), which gives us a stable kind without an exhaustive
// switch over every `TriviaPiece` case.
let kind = Mirror(reflecting: piece).children.first?.label ?? "\(piece)"
if keptTriviaKinds.contains(kind) {
result.append([
"kind": kind,
"text": Trivia(pieces: [piece]).description,
"range": [
"start": location(AbsolutePosition(utf8Offset: offset), converter),
"end": location(AbsolutePosition(utf8Offset: offset + length), converter),
],
])
}
offset += length
}
return result
}
/// Recursively convert a SwiftSyntax node into a JSON-serializable value.
///
/// * Tokens carry `kind`, `tokenKind`, `text`, and `range`, plus
/// `leadingTrivia`/`trailingTrivia` — but only when non-empty (after
/// filtering, most tokens have no trivia, so the keys are simply absent).
/// * Layout nodes (e.g. `functionDecl`) carry `kind` and source `range`, and
/// additionally embed their children directly as members keyed by the
/// child's name in the parent (e.g. `name`, `signature`, `body`); absent
/// optional children are omitted. Field names never collide with
/// `kind`/`range`.
/// * Collection nodes (e.g. `codeBlockItemList`) are *elided*: they become a
/// plain array of their serialized elements, taking the place of the
/// collection node itself. A list-valued layout field (e.g. `parameters`) is
/// therefore simply a JSON array. This drops the collection node's own
/// `kind`/`range`, which are unnamed and largely recoverable from the
/// elements.
private func serialize(
_ node: Syntax,
_ converter: SourceLocationConverter
) -> Any {
if node.kind.isSyntaxCollection {
return node.children(viewMode: .sourceAccurate).map {
serialize($0, converter)
}
}
// Source range covering the node's content, excluding surrounding trivia.
let range: [String: Any] = [
"start": location(node.positionAfterSkippingLeadingTrivia, converter),
"end": location(node.endPositionBeforeTrailingTrivia, converter),
]
if let token = node.as(TokenSyntax.self) {
var result: [String: Any] = [
"kind": "token",
"tokenKind": "\(token.tokenKind)",
"text": token.text,
"range": range,
]
// Only emit trivia when present; after filtering, most tokens have none.
// Leading trivia starts at the token's own position; trailing trivia
// starts just after the token's content.
let leading = serializeTrivia(
token.leadingTrivia, startingAt: token.position, converter)
if !leading.isEmpty {
result["leadingTrivia"] = leading
}
let trailing = serializeTrivia(
token.trailingTrivia,
startingAt: token.endPositionBeforeTrailingTrivia,
converter)
if !trailing.isEmpty {
result["trailingTrivia"] = trailing
}
return result
}
var result: [String: Any] = [
"kind": "\(node.kind)",
"range": range,
]
var unnamed = 0
for child in node.children(viewMode: .sourceAccurate) {
// Layout children are named; embed each under its field name in the
// parent (the same mechanism SwiftSyntax uses for its debug dump). A
// child that is a collection serializes to an array (see above).
if let keyPath = child.keyPathInParent, let name = childName(keyPath) {
result[name] = serialize(child, converter)
} else {
// Defensive fallback for any unnamed layout child.
result["child\(unnamed)"] = serialize(child, converter)
unnamed += 1
}
}
return result
}
/// Parse the given NUL-terminated Swift source string and return a
/// heap-allocated, NUL-terminated JSON representation of the syntax tree.
///
/// The returned pointer is owned by the caller and MUST be released with
/// `ssr_string_free`. Returns `nil` on failure.
@_cdecl("ssr_parse_json")
public func ssr_parse_json(_ source: UnsafePointer<CChar>?) -> UnsafeMutablePointer<CChar>? {
guard let source = source else { return nil }
let code = String(cString: source)
let tree = Parser.parse(source: code)
let converter = SourceLocationConverter(fileName: "<input>", tree: tree)
let json = serialize(Syntax(tree), converter)
guard
let data = try? JSONSerialization.data(
withJSONObject: json, options: [.sortedKeys]),
let string = String(data: data, encoding: .utf8)
else {
return nil
}
return strdup(string)
}
/// Free a string previously returned by this library.
@_cdecl("ssr_string_free")
public func ssr_string_free(_ ptr: UnsafeMutablePointer<CChar>?) {
free(ptr)
}

19
unified/swift-syntax-rs/swift_runtime.bzl Normal file
View File

@@ -0,0 +1,19 @@
"""Filter the Swift toolchain down to just its Linux runtime shared objects.
The standalone toolchain's `:files` bundles the compiler, host libraries, clang
runtime, plugins, etc. For running a Swift-linked binary we only need the
runtime shared objects in `usr/lib/swift/linux/`.
"""
def _swift_runtime_libs_impl(ctx):
libs = [
f
for f in ctx.files.toolchain
if "/usr/lib/swift/linux/" in f.path and f.path.endswith(".so")
]
return [DefaultInfo(files = depset(libs))]
swift_runtime_libs = rule(
implementation = _swift_runtime_libs_impl,
attrs = {"toolchain": attr.label(allow_files = True)},
)

92
unified/swift-syntax-rs/xcode_transition.bzl Normal file
View File

@@ -0,0 +1,92 @@
"""Per-target Xcode configuration for `//unified/swift-syntax-rs` on macOS.
`rules_swift`'s auto-registered `xcode_swift_toolchain` reads
`cc_toolchain.target_gnu_system_name`, which is literally "local" on
Bazel's built-in `local_config_cc`. To make it usable we need:
- `--xcode_version_config=@local_config_xcode//:host_xcodes` — selects
`apple_support`'s xcode_config (matches `rules_swift`'s `system_sdk` keys).
- `--extra_toolchains=@local_config_apple_cc_toolchains//:all` — forces
`apple_support`'s CC toolchain ahead of `local_config_cc`.
Applied via an incoming-edge Starlark transition on the `swift_library`
target only (via the `xcode_transition_swift_library` macro), so downstream
Rust targets and the rest of the repo stay on the default CC toolchain and
the `@local_config_*` repos are not materialized otherwise. No-op off macOS.
"""
load("@rules_cc//cc/common:cc_info.bzl", "CcInfo")
load("@rules_swift//swift:swift.bzl", "swift_library")
load("//misc/bazel:os.bzl", "os_select")
_XCODE_VERSION_CONFIG = "//command_line_option:xcode_version_config"
_EXTRA_TOOLCHAINS = "//command_line_option:extra_toolchains"
def _transition_impl(settings, attr):
if attr.os != "macos":
return {}
else:
return {
_XCODE_VERSION_CONFIG: "@local_config_xcode//:host_xcodes",
_EXTRA_TOOLCHAINS: (
list(settings[_EXTRA_TOOLCHAINS]) +
["@local_config_apple_cc_toolchains//:all"]
),
}
_xcode_transition = transition(
implementation = _transition_impl,
inputs = [_EXTRA_TOOLCHAINS],
outputs = [_XCODE_VERSION_CONFIG, _EXTRA_TOOLCHAINS],
)
def _wrapper_impl(ctx):
src = ctx.attr.actual[0]
# Forward the providers a downstream `rust_*` target reads from `deps`:
# `DefaultInfo`, `CcInfo` (linking info), and `OutputGroupInfo`.
providers = [src[DefaultInfo]]
for p in (CcInfo, OutputGroupInfo):
if p in src:
providers.append(src[p])
return providers
_xcode_transition_swift_library_rule = rule(
implementation = _wrapper_impl,
attrs = {
"actual": attr.label(
mandatory = True,
cfg = _xcode_transition,
providers = [CcInfo],
),
"os": attr.string(mandatory = True),
"_allowlist_function_transition": attr.label(
default = "@bazel_tools//tools/allowlists/function_transition_allowlist",
),
},
)
def xcode_transition_swift_library(name, visibility = None, tags = None, target_compatible_with = None, **kwargs):
"""`swift_library` wrapped in the macOS Xcode-config transition.
Emits a private inner `_impl_<name>` `swift_library` (tagged `manual`)
and a public `name` that applies the transition on macOS and forwards
the inner target's providers. Downstream `rust_*` targets depend on
`name` as usual; only the `swift_library` sub-graph flips toolchain.
"""
inner_name = "_impl_%s" % name
swift_library(
name = inner_name,
visibility = ["//visibility:private"],
tags = (tags or []) + ["manual"],
target_compatible_with = target_compatible_with,
**kwargs
)
_xcode_transition_swift_library_rule(
name = name,
visibility = visibility,
tags = tags,
target_compatible_with = target_compatible_with,
actual = ":" + inner_name,
os = os_select(linux = "linux", macos = "macos", default = "other"),
)