Merge pull request #21812 from asgerf/asgerf/swift-yeast-1

Add tree-sitter-swift extractor scaffolding and YEAST desugaring
2026-05-14 11:19:27 +02:00 · 2026-05-08 13:21:17 +02:00
parent 81e1ab7aab 33e89ea123
commit fdef477138
40 changed files with 6566 additions and 0 deletions
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -438,6 +438,25 @@ dependencies = [
 "tree-sitter-ruby",
 ]

+[[package]]
+name = "codeql-extractor-unified"
+version = "0.1.0"
+dependencies = [
+ "clap",
+ "codeql-extractor",
+ "encoding",
+ "lazy_static",
+ "rayon",
+ "regex",
+ "serde_json",
+ "tracing",
+ "tracing-subscriber",
+ "tree-sitter",
+ "tree-sitter-embedded-template",
+ "tree-sitter-swift",
+ "yeast",
+]
+
 [[package]]
 name = "codeql-rust"
 version = "0.1.0"
@@ -2922,6 +2941,16 @@ dependencies = [
 "tree-sitter-language",
 ]

+[[package]]
+name = "tree-sitter-swift"
+version = "0.7.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f3b98fb6bc8e6a6a10023f401aa6a1858115e849dfaf7de57dd8b8ea0f257bd9"
+dependencies = [
+ "cc",
+ "tree-sitter-language",
+]
+
 [[package]]
 name = "triomphe"
 version = "0.1.14"
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -7,6 +7,7 @@ members = [
    "shared/yeast",
    "shared/yeast-macros",
    "ruby/extractor",
+    "unified/extractor",
    "rust/extractor",
    "rust/extractor/macros",
    "rust/ast-generator",
--- a/MODULE.bazel
+++ b/MODULE.bazel
@@ -153,6 +153,7 @@ use_repo(
    "vendor_ts__tree-sitter-python-0.23.6",
    "vendor_ts__tree-sitter-ql-0.23.1",
    "vendor_ts__tree-sitter-ruby-0.23.1",
+    "vendor_ts__tree-sitter-swift-0.7.2",
    "vendor_ts__triomphe-0.1.14",
    "vendor_ts__ungrammar-1.16.1",
    "vendor_ts__zstd-0.13.3",
--- a/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.bazel
+++ b/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.bazel
@@ -673,6 +673,18 @@ alias(
    tags = ["manual"],
 )

+alias(
+    name = "tree-sitter-swift-0.7.2",
+    actual = "@vendor_ts__tree-sitter-swift-0.7.2//:tree_sitter_swift",
+    tags = ["manual"],
+)
+
+alias(
+    name = "tree-sitter-swift",
+    actual = "@vendor_ts__tree-sitter-swift-0.7.2//:tree_sitter_swift",
+    tags = ["manual"],
+)
+
 alias(
    name = "triomphe-0.1.14",
    actual = "@vendor_ts__triomphe-0.1.14//:triomphe",
--- a/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.tree-sitter-swift-0.7.2.bazel
+++ b/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.tree-sitter-swift-0.7.2.bazel
@@ -0,0 +1,166 @@
+###############################################################################
+# @generated
+# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To
+# regenerate this file, run the following:
+#
+#     bazel run @@//misc/bazel/3rdparty:vendor_tree_sitter_extractors
+###############################################################################
+
+load(
+    "@rules_rust//cargo:defs.bzl",
+    "cargo_build_script",
+    "cargo_toml_env_vars",
+)
+load("@rules_rust//rust:defs.bzl", "rust_library")
+
+package(default_visibility = ["//visibility:public"])
+
+cargo_toml_env_vars(
+    name = "cargo_toml_env_vars",
+    src = "Cargo.toml",
+)
+
+rust_library(
+    name = "tree_sitter_swift",
+    srcs = glob(
+        include = ["**/*.rs"],
+        allow_empty = True,
+    ),
+    compile_data = glob(
+        include = ["**"],
+        allow_empty = True,
+        exclude = [
+            "**/* *",
+            ".tmp_git_root/**/*",
+            "BUILD",
+            "BUILD.bazel",
+            "WORKSPACE",
+            "WORKSPACE.bazel",
+        ],
+    ),
+    crate_root = "bindings/rust/lib.rs",
+    edition = "2018",
+    rustc_env_files = [
+        ":cargo_toml_env_vars",
+    ],
+    rustc_flags = [
+        "--cap-lints=allow",
+    ],
+    tags = [
+        "cargo-bazel",
+        "crate-name=tree-sitter-swift",
+        "manual",
+        "noclippy",
+        "norustfmt",
+    ],
+    target_compatible_with = select({
+        "@rules_rust//rust/platform:aarch64-apple-darwin": [],
+        "@rules_rust//rust/platform:aarch64-apple-ios": [],
+        "@rules_rust//rust/platform:aarch64-apple-ios-sim": [],
+        "@rules_rust//rust/platform:aarch64-linux-android": [],
+        "@rules_rust//rust/platform:aarch64-pc-windows-msvc": [],
+        "@rules_rust//rust/platform:aarch64-unknown-fuchsia": [],
+        "@rules_rust//rust/platform:aarch64-unknown-linux-gnu": [],
+        "@rules_rust//rust/platform:aarch64-unknown-nixos-gnu": [],
+        "@rules_rust//rust/platform:aarch64-unknown-nto-qnx710": [],
+        "@rules_rust//rust/platform:aarch64-unknown-uefi": [],
+        "@rules_rust//rust/platform:arm-unknown-linux-gnueabi": [],
+        "@rules_rust//rust/platform:arm-unknown-linux-musleabi": [],
+        "@rules_rust//rust/platform:armv7-linux-androideabi": [],
+        "@rules_rust//rust/platform:armv7-unknown-linux-gnueabi": [],
+        "@rules_rust//rust/platform:i686-apple-darwin": [],
+        "@rules_rust//rust/platform:i686-linux-android": [],
+        "@rules_rust//rust/platform:i686-pc-windows-msvc": [],
+        "@rules_rust//rust/platform:i686-unknown-freebsd": [],
+        "@rules_rust//rust/platform:i686-unknown-linux-gnu": [],
+        "@rules_rust//rust/platform:powerpc-unknown-linux-gnu": [],
+        "@rules_rust//rust/platform:riscv32imc-unknown-none-elf": [],
+        "@rules_rust//rust/platform:riscv64gc-unknown-linux-gnu": [],
+        "@rules_rust//rust/platform:riscv64gc-unknown-none-elf": [],
+        "@rules_rust//rust/platform:s390x-unknown-linux-gnu": [],
+        "@rules_rust//rust/platform:thumbv7em-none-eabi": [],
+        "@rules_rust//rust/platform:thumbv8m.main-none-eabi": [],
+        "@rules_rust//rust/platform:wasm32-unknown-emscripten": [],
+        "@rules_rust//rust/platform:wasm32-unknown-unknown": [],
+        "@rules_rust//rust/platform:wasm32-wasip1": [],
+        "@rules_rust//rust/platform:wasm32-wasip1-threads": [],
+        "@rules_rust//rust/platform:wasm32-wasip2": [],
+        "@rules_rust//rust/platform:x86_64-apple-darwin": [],
+        "@rules_rust//rust/platform:x86_64-apple-ios": [],
+        "@rules_rust//rust/platform:x86_64-linux-android": [],
+        "@rules_rust//rust/platform:x86_64-pc-windows-msvc": [],
+        "@rules_rust//rust/platform:x86_64-unknown-freebsd": [],
+        "@rules_rust//rust/platform:x86_64-unknown-fuchsia": [],
+        "@rules_rust//rust/platform:x86_64-unknown-linux-gnu": [],
+        "@rules_rust//rust/platform:x86_64-unknown-nixos-gnu": [],
+        "@rules_rust//rust/platform:x86_64-unknown-none": [],
+        "@rules_rust//rust/platform:x86_64-unknown-uefi": [],
+        "//conditions:default": ["@platforms//:incompatible"],
+    }),
+    version = "0.7.2",
+    deps = [
+        "@vendor_ts__tree-sitter-language-0.1.5//:tree_sitter_language",
+        "@vendor_ts__tree-sitter-swift-0.7.2//:build_script_build",
+    ],
+)
+
+cargo_build_script(
+    name = "_bs",
+    srcs = glob(
+        include = ["**/*.rs"],
+        allow_empty = True,
+    ),
+    compile_data = glob(
+        include = ["**"],
+        allow_empty = True,
+        exclude = [
+            "**/* *",
+            "**/*.rs",
+            ".tmp_git_root/**/*",
+            "BUILD",
+            "BUILD.bazel",
+            "WORKSPACE",
+            "WORKSPACE.bazel",
+        ],
+    ),
+    crate_name = "build_script_build",
+    crate_root = "bindings/rust/build.rs",
+    data = glob(
+        include = ["**"],
+        allow_empty = True,
+        exclude = [
+            "**/* *",
+            ".tmp_git_root/**/*",
+            "BUILD",
+            "BUILD.bazel",
+            "WORKSPACE",
+            "WORKSPACE.bazel",
+        ],
+    ),
+    edition = "2018",
+    pkg_name = "tree-sitter-swift",
+    rustc_env_files = [
+        ":cargo_toml_env_vars",
+    ],
+    rustc_flags = [
+        "--cap-lints=allow",
+    ],
+    tags = [
+        "cargo-bazel",
+        "crate-name=tree-sitter-swift",
+        "manual",
+        "noclippy",
+        "norustfmt",
+    ],
+    version = "0.7.2",
+    visibility = ["//visibility:private"],
+    deps = [
+        "@vendor_ts__cc-1.2.61//:cc",
+    ],
+)
+
+alias(
+    name = "build_script_build",
+    actual = ":_bs",
+    tags = ["manual"],
+)
--- a/misc/bazel/3rdparty/tree_sitter_extractors_deps/defs.bzl
+++ b/misc/bazel/3rdparty/tree_sitter_extractors_deps/defs.bzl
@@ -403,6 +403,21 @@ _NORMAL_DEPENDENCIES = {
            "syn": Label("@vendor_ts__syn-2.0.106//:syn"),
        },
    },
+    "unified/extractor": {
+        _COMMON_CONDITION: {
+            "clap": Label("@vendor_ts__clap-4.5.48//:clap"),
+            "encoding": Label("@vendor_ts__encoding-0.2.33//:encoding"),
+            "lazy_static": Label("@vendor_ts__lazy_static-1.5.0//:lazy_static"),
+            "rayon": Label("@vendor_ts__rayon-1.11.0//:rayon"),
+            "regex": Label("@vendor_ts__regex-1.11.3//:regex"),
+            "serde_json": Label("@vendor_ts__serde_json-1.0.145//:serde_json"),
+            "tracing": Label("@vendor_ts__tracing-0.1.41//:tracing"),
+            "tracing-subscriber": Label("@vendor_ts__tracing-subscriber-0.3.20//:tracing_subscriber"),
+            "tree-sitter": Label("@vendor_ts__tree-sitter-0.26.8//:tree_sitter"),
+            "tree-sitter-embedded-template": Label("@vendor_ts__tree-sitter-embedded-template-0.25.0//:tree_sitter_embedded_template"),
+            "tree-sitter-swift": Label("@vendor_ts__tree-sitter-swift-0.7.2//:tree_sitter_swift"),
+        },
+    },
 }

 _NORMAL_ALIASES = {
@@ -437,6 +452,10 @@ _NORMAL_ALIASES = {
        _COMMON_CONDITION: {
        },
    },
+    "unified/extractor": {
+        _COMMON_CONDITION: {
+        },
+    },
 }

 _NORMAL_DEV_DEPENDENCIES = {
@@ -461,6 +480,8 @@ _NORMAL_DEV_DEPENDENCIES = {
    },
    "shared/yeast-macros": {
    },
+    "unified/extractor": {
+    },
 }

 _NORMAL_DEV_ALIASES = {
@@ -482,6 +503,8 @@ _NORMAL_DEV_ALIASES = {
    },
    "shared/yeast-macros": {
    },
+    "unified/extractor": {
+    },
 }

 _PROC_MACRO_DEPENDENCIES = {
@@ -501,6 +524,8 @@ _PROC_MACRO_DEPENDENCIES = {
    },
    "shared/yeast-macros": {
    },
+    "unified/extractor": {
+    },
 }

 _PROC_MACRO_ALIASES = {
@@ -520,6 +545,8 @@ _PROC_MACRO_ALIASES = {
    },
    "shared/yeast-macros": {
    },
+    "unified/extractor": {
+    },
 }

 _PROC_MACRO_DEV_DEPENDENCIES = {
@@ -539,6 +566,8 @@ _PROC_MACRO_DEV_DEPENDENCIES = {
    },
    "shared/yeast-macros": {
    },
+    "unified/extractor": {
+    },
 }

 _PROC_MACRO_DEV_ALIASES = {
@@ -560,6 +589,8 @@ _PROC_MACRO_DEV_ALIASES = {
    },
    "shared/yeast-macros": {
    },
+    "unified/extractor": {
+    },
 }

 _BUILD_DEPENDENCIES = {
@@ -579,6 +610,8 @@ _BUILD_DEPENDENCIES = {
    },
    "shared/yeast-macros": {
    },
+    "unified/extractor": {
+    },
 }

 _BUILD_ALIASES = {
@@ -598,6 +631,8 @@ _BUILD_ALIASES = {
    },
    "shared/yeast-macros": {
    },
+    "unified/extractor": {
+    },
 }

 _BUILD_PROC_MACRO_DEPENDENCIES = {
@@ -617,6 +652,8 @@ _BUILD_PROC_MACRO_DEPENDENCIES = {
    },
    "shared/yeast-macros": {
    },
+    "unified/extractor": {
+    },
 }

 _BUILD_PROC_MACRO_ALIASES = {
@@ -636,6 +673,8 @@ _BUILD_PROC_MACRO_ALIASES = {
    },
    "shared/yeast-macros": {
    },
+    "unified/extractor": {
+    },
 }

 _CONDITIONS = {
@@ -3497,6 +3536,16 @@ def crate_repositories():
        build_file = Label("//misc/bazel/3rdparty/tree_sitter_extractors_deps:BUILD.tree-sitter-ruby-0.23.1.bazel"),
    )

+    maybe(
+        http_archive,
+        name = "vendor_ts__tree-sitter-swift-0.7.2",
+        sha256 = "f3b98fb6bc8e6a6a10023f401aa6a1858115e849dfaf7de57dd8b8ea0f257bd9",
+        type = "tar.gz",
+        urls = ["https://static.crates.io/crates/tree-sitter-swift/0.7.2/download"],
+        strip_prefix = "tree-sitter-swift-0.7.2",
+        build_file = Label("//misc/bazel/3rdparty/tree_sitter_extractors_deps:BUILD.tree-sitter-swift-0.7.2.bazel"),
+    )
+
    maybe(
        http_archive,
        name = "vendor_ts__triomphe-0.1.14",
@@ -4238,6 +4287,7 @@ def crate_repositories():
        struct(repo = "vendor_ts__tree-sitter-embedded-template-0.25.0", is_dev_dep = False),
        struct(repo = "vendor_ts__tree-sitter-python-0.23.6", is_dev_dep = False),
        struct(repo = "vendor_ts__tree-sitter-ruby-0.23.1", is_dev_dep = False),
+        struct(repo = "vendor_ts__tree-sitter-swift-0.7.2", is_dev_dep = False),
        struct(repo = "vendor_ts__triomphe-0.1.14", is_dev_dep = False),
        struct(repo = "vendor_ts__ungrammar-1.16.1", is_dev_dep = False),
        struct(repo = "vendor_ts__zstd-0.13.3", is_dev_dep = False),
--- a/unified/.gitignore
+++ b/unified/.gitignore
@@ -0,0 +1,6 @@
+target
+.vscode/launch.json
+.cache
+ql/test/**/*.testproj
+ql/test/**/*.actual
+.codeql
--- a/unified/AGENTS.md
+++ b/unified/AGENTS.md
@@ -0,0 +1,15 @@
+# Agent instructions
+
+This is a CodeQL extractor based on tree-sitter.
+
+## Building
+To build the extractor, run `scripts/create-extractor-pack.sh`
+
+## Testing
+- If you changed the extractor code, always rebuild it before running tests.
+
+- To run all tests, run `codeql test run --search-path extractor-pack ql/test`
+
+- Do not edit `.expected` files manually. To update the expected output, pass `--learn` to the `codeql test run` command.
+
+- To run a specific test, pass the specific directory to the `codeql test run` command instead of `ql/test`.
--- a/unified/BUILD.bazel
+++ b/unified/BUILD.bazel
@@ -0,0 +1,57 @@
+load("@rules_pkg//pkg:mappings.bzl", "pkg_filegroup")
+load("//misc/bazel:pkg.bzl", "codeql_pack", "codeql_pkg_files")
+
+package(default_visibility = ["//visibility:public"])
+
+alias(
+    name = "dbscheme",
+    actual = "//unified/ql/lib:dbscheme",
+)
+
+alias(
+    name = "dbscheme-stats",
+    actual = "//unified/ql/lib:dbscheme-stats",
+)
+
+codeql_pkg_files(
+    name = "dbscheme-group",
+    srcs = [
+        ":dbscheme",
+        ":dbscheme-stats",
+    ],
+    strip_prefix = None,
+)
+
+pkg_filegroup(
+    name = "db-files",
+    srcs = [
+        ":dbscheme-group",
+    ],
+)
+
+codeql_pkg_files(
+    name = "codeql-extractor-yml",
+    srcs = [
+        "codeql-extractor.yml",
+        "//:LICENSE",
+    ],
+    strip_prefix = None,
+)
+
+codeql_pkg_files(
+    name = "extractor-arch",
+    exes = [
+        "//unified/extractor",
+    ],
+    prefix = "tools/{CODEQL_PLATFORM}",
+)
+
+codeql_pack(
+    name = "unified",
+    srcs = [
+        ":codeql-extractor-yml",
+        ":dbscheme-group",
+        ":extractor-arch",
+        "//unified/tools",
+    ],
+)
--- a/unified/codeql-extractor.yml
+++ b/unified/codeql-extractor.yml
@@ -0,0 +1,17 @@
+name: "unified"
+display_name: "Unified Language"
+version: 0.0.1
+column_kind: "utf8"
+legacy_qltest_extraction: true
+build_modes:
+  - none
+github_api_languages:
+  - Swift
+scc_languages:
+  - Swift
+file_types:
+  - name: swift
+    display_name: Swift files
+    extensions:
+       - .swift
+       - .swiftinterface
--- a/unified/extractor/BUILD.bazel
+++ b/unified/extractor/BUILD.bazel
@@ -0,0 +1,20 @@
+load("//misc/bazel:rust.bzl", "codeql_rust_binary")
+load("//misc/bazel/3rdparty/tree_sitter_extractors_deps:defs.bzl", "aliases", "all_crate_deps")
+
+exports_files(["Cargo.toml"])
+
+codeql_rust_binary(
+    name = "extractor",
+    srcs = glob(["src/**/*.rs"]),
+    aliases = aliases(),
+    proc_macro_deps = all_crate_deps(
+        proc_macro = True,
+    ),
+    visibility = ["//visibility:public"],
+    deps = all_crate_deps(
+        normal = True,
+    ) + [
+        "//shared/tree-sitter-extractor",
+        "//shared/yeast",
+    ],
+)
--- a/unified/extractor/Cargo.toml
+++ b/unified/extractor/Cargo.toml
@@ -0,0 +1,23 @@
+[package]
+name = "codeql-extractor-unified"
+description = "CodeQL Unified extractor"
+version = "0.1.0"
+authors = ["GitHub"]
+edition = "2024"
+
+# When updating these dependencies, run `misc/bazel/3rdparty/update_cargo_deps.sh`
+[dependencies]
+tree-sitter = ">= 0.23.0"
+tree-sitter-embedded-template = "0.25.0"
+tree-sitter-swift = "0.7.2"
+clap = { version = "4.5", features = ["derive"] }
+tracing = "0.1"
+tracing-subscriber = { version = "0.3.20", features = ["env-filter"] }
+rayon = "1.11.0"
+regex = "1.11.3"
+encoding = "0.2"
+lazy_static = "1.5.0"
+serde_json = "1.0.145"
+
+codeql-extractor = { path = "../../shared/tree-sitter-extractor" }
+yeast = { path = "../../shared/yeast" }
--- a/unified/extractor/src/autobuilder.rs
+++ b/unified/extractor/src/autobuilder.rs
@@ -0,0 +1,20 @@
+use std::env;
+use std::path::PathBuf;
+
+use clap::Args;
+
+use codeql_extractor::autobuilder;
+
+#[derive(Args)]
+// The autobuilder takes no command-line options, but this may change in the future.
+pub struct Options {}
+
+pub fn run(_: Options) -> std::io::Result<()> {
+    let database = env::var("CODEQL_EXTRACTOR_UNIFIED_WIP_DATABASE")
+        .expect("CODEQL_EXTRACTOR_UNIFIED_WIP_DATABASE not set");
+
+    autobuilder::Autobuilder::new("unified", PathBuf::from(database))
+        .include_extensions(&[".swift", ".swiftinterface"])
+        .size_limit("10m")
+        .run()
+}
--- a/unified/extractor/src/extractor.rs
+++ b/unified/extractor/src/extractor.rs
@@ -0,0 +1,40 @@
+use clap::Args;
+use std::path::PathBuf;
+
+use codeql_extractor::extractor::simple;
+use codeql_extractor::trap;
+
+#[path = "languages/swift/swift.rs"]
+mod swift;
+
+#[derive(Args)]
+pub struct Options {
+    /// Sets a custom source archive folder
+    #[arg(long)]
+    source_archive_dir: PathBuf,
+
+    /// Sets a custom trap folder
+    #[arg(long)]
+    output_dir: PathBuf,
+
+    /// A text file containing the paths of the files to extract
+    #[arg(long)]
+    file_list: PathBuf,
+}
+
+pub fn run(options: Options) -> std::io::Result<()> {
+    codeql_extractor::extractor::set_tracing_level("unified");
+
+    let extractor = simple::Extractor {
+        prefix: "unified".to_string(),
+        languages: vec![
+            swift::language_spec(),
+        ],
+        trap_dir: options.output_dir,
+        trap_compression: trap::Compression::from_env("CODEQL_EXTRACTOR_UNIFIED_OPTION_TRAP_COMPRESSION"),
+        source_archive_dir: options.source_archive_dir,
+        file_lists: vec![options.file_list],
+    };
+
+    extractor.run()
+}
--- a/unified/extractor/src/generator.rs
+++ b/unified/extractor/src/generator.rs
@@ -0,0 +1,27 @@
+use clap::Args;
+use std::path::PathBuf;
+
+use codeql_extractor::generator::{generate, language::Language};
+
+#[derive(Args)]
+pub struct Options {
+    /// Path of the generated dbscheme file
+    #[arg(long)]
+    dbscheme: PathBuf,
+
+    /// Path of the generated QLL file
+    #[arg(long)]
+    library: PathBuf,
+}
+
+pub fn run(options: Options) -> std::io::Result<()> {
+    codeql_extractor::extractor::set_tracing_level("unified");
+
+    let languages = vec![Language {
+        name: "Swift".to_owned(),
+        node_types: tree_sitter_swift::NODE_TYPES,
+        desugar: None,
+    }];
+
+    generate(languages, options.dbscheme, options.library, "run unified/scripts/create-extractor-pack.sh")
+}
--- a/unified/extractor/src/languages/swift/swift.rs
+++ b/unified/extractor/src/languages/swift/swift.rs
@@ -0,0 +1,23 @@
+use codeql_extractor::extractor::simple;
+use yeast::{rule, DesugaringConfig};
+
+fn desugaring_rules() -> Vec<yeast::Rule> {
+    vec![
+        rule!(
+            (additive_expression)
+            =>
+            (simple_identifier "blah")
+        ),
+    ]
+}
+
+pub fn language_spec() -> simple::LanguageSpec {
+    let desugar = DesugaringConfig::new().add_phase("desugar", desugaring_rules());
+    simple::LanguageSpec {
+        prefix: "swift",
+        ts_language: tree_sitter_swift::LANGUAGE.into(),
+        node_types: tree_sitter_swift::NODE_TYPES,
+        file_globs: vec!["*.swift".into(), "*.swiftinterface".into()],
+        desugar: Some(desugar),
+    }
+}
--- a/unified/extractor/src/main.rs
+++ b/unified/extractor/src/main.rs
@@ -0,0 +1,23 @@
+use clap::Parser;
+
+mod autobuilder;
+mod extractor;
+mod generator;
+
+#[derive(Parser)]
+#[command(author, version, about)]
+enum Cli {
+    Extract(extractor::Options),
+    Generate(generator::Options),
+    Autobuild(autobuilder::Options),
+}
+
+fn main() -> std::io::Result<()> {
+    let cli = Cli::parse();
+
+    match cli {
+        Cli::Extract(options) => extractor::run(options),
+        Cli::Generate(options) => generator::run(options),
+        Cli::Autobuild(options) => autobuilder::run(options),
+    }
+}
--- a/unified/ql/lib/BUILD.bazel
+++ b/unified/ql/lib/BUILD.bazel
@@ -0,0 +1,13 @@
+load("@rules_pkg//pkg:mappings.bzl", "pkg_files")
+
+package(default_visibility = ["//unified:__pkg__"])
+
+pkg_files(
+    name = "dbscheme",
+    srcs = ["unified.dbscheme"],
+)
+
+pkg_files(
+    name = "dbscheme-stats",
+    srcs = ["unified.dbscheme.stats"],
+)
--- a/unified/ql/lib/codeql/Locations.qll
+++ b/unified/ql/lib/codeql/Locations.qll
@@ -0,0 +1,70 @@
+/** Provides classes for working with locations. */
+overlay[local]
+module;
+
+import files.FileSystem
+
+/**
+ * A location as given by a file, a start line, a start column,
+ * an end line, and an end column.
+ *
+ * For more information about locations see [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/).
+ */
+class Location extends @location_default {
+  /** Gets the file for this location. */
+  File getFile() { locations_default(this, result, _, _, _, _) }
+
+  /** Gets the 1-based line number (inclusive) where this location starts. */
+  int getStartLine() { locations_default(this, _, result, _, _, _) }
+
+  /** Gets the 1-based column number (inclusive) where this location starts. */
+  int getStartColumn() { locations_default(this, _, _, result, _, _) }
+
+  /** Gets the 1-based line number (inclusive) where this location ends. */
+  int getEndLine() { locations_default(this, _, _, _, result, _) }
+
+  /** Gets the 1-based column number (inclusive) where this location ends. */
+  int getEndColumn() { locations_default(this, _, _, _, _, result) }
+
+  /** Gets the number of lines covered by this location. */
+  int getNumLines() { result = this.getEndLine() - this.getStartLine() + 1 }
+
+  /** Gets a textual representation of this element. */
+  bindingset[this]
+  pragma[inline_late]
+  string toString() {
+    exists(string filepath, int startline, int startcolumn, int endline, int endcolumn |
+      this.hasLocationInfo(filepath, startline, startcolumn, endline, endcolumn) and
+      result = filepath + "@" + startline + ":" + startcolumn + ":" + endline + ":" + endcolumn
+    )
+  }
+
+  /**
+   * Holds if this element is at the specified location.
+   * The location spans column `startcolumn` of line `startline` to
+   * column `endcolumn` of line `endline` in file `filepath`.
+   * For more information, see
+   * [Providing locations in CodeQL queries](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/).
+   */
+  predicate hasLocationInfo(
+    string filepath, int startline, int startcolumn, int endline, int endcolumn
+  ) {
+    exists(File f |
+      locations_default(this, f, startline, startcolumn, endline, endcolumn) and
+      filepath = f.getAbsolutePath()
+    )
+  }
+
+  /** Holds if this location starts strictly before the specified location. */
+  pragma[inline]
+  predicate strictlyBefore(Location other) {
+    this.getStartLine() < other.getStartLine()
+    or
+    this.getStartLine() = other.getStartLine() and this.getStartColumn() < other.getStartColumn()
+  }
+}
+
+/** An entity representing an empty location. */
+class EmptyLocation extends Location {
+  EmptyLocation() { empty_location(this) }
+}
--- a/unified/ql/lib/codeql/files/FileSystem.qll
+++ b/unified/ql/lib/codeql/files/FileSystem.qll
@@ -0,0 +1,38 @@
+/** Provides classes for working with files and folders. */
+overlay[local]
+module;
+
+private import codeql.Locations
+private import codeql.util.FileSystem
+
+private module Input implements InputSig {
+  abstract class ContainerBase extends @container {
+    abstract string getAbsolutePath();
+
+    ContainerBase getParentContainer() { containerparent(result, this) }
+
+    string toString() { result = this.getAbsolutePath() }
+  }
+
+  class FolderBase extends ContainerBase, @folder {
+    override string getAbsolutePath() { folders(this, result) }
+  }
+
+  class FileBase extends ContainerBase, @file {
+    override string getAbsolutePath() { files(this, result) }
+  }
+
+  predicate hasSourceLocationPrefix = sourceLocationPrefix/1;
+}
+
+private module Impl = Make<Input>;
+
+class Container = Impl::Container;
+
+class Folder = Impl::Folder;
+
+/** A file. */
+class File extends Container, Impl::File {
+  /** Holds if this file was extracted from ordinary source code. */
+  predicate fromSource() { any() }
+}
--- a/unified/ql/lib/codeql/unified/Ast.qll
+++ b/unified/ql/lib/codeql/unified/Ast.qll
--- a/unified/ql/lib/qlpack.lock.yml
+++ b/unified/ql/lib/qlpack.lock.yml
@@ -0,0 +1,4 @@
+---
+dependencies: {}
+compiled: false
+lockVersion: 1.0.0
--- a/unified/ql/lib/qlpack.yml
+++ b/unified/ql/lib/qlpack.yml
@@ -0,0 +1,11 @@
+name: codeql/unified-all
+version: 0.0.1-dev
+groups: unified
+dbscheme: unified.dbscheme
+extractor: unified
+library: true
+upgrades: upgrades
+dependencies:
+  codeql/util: ${workspace}
+warnOnImplicitThis: true
+compileForOverlayEval: true
--- a/unified/ql/lib/unified.dbscheme
+++ b/unified/ql/lib/unified.dbscheme
--- a/unified/ql/lib/unified.dbscheme.stats
+++ b/unified/ql/lib/unified.dbscheme.stats
@@ -0,0 +1,4 @@
+<dbstats>
+  <typesizes />
+  <stats />
+</dbstats>
--- a/unified/ql/src/qlpack.lock.yml
+++ b/unified/ql/src/qlpack.lock.yml
@@ -0,0 +1,4 @@
+---
+dependencies: {}
+compiled: false
+lockVersion: 1.0.0
--- a/unified/ql/src/qlpack.yml
+++ b/unified/ql/src/qlpack.yml
@@ -0,0 +1,11 @@
+name: codeql/unified-queries
+version: 0.0.1-dev
+groups:
+  - unified
+  - queries
+suites: codeql-suites
+dependencies:
+  codeql/unified-all: ${workspace}
+  codeql/suite-helpers: ${workspace}
+  codeql/util: ${workspace}
+warnOnImplicitThis: true
--- a/unified/ql/test/library-tests/BasicTest/test.expected
+++ b/unified/ql/test/library-tests/BasicTest/test.expected
@@ -0,0 +1,101 @@
+identifier
+| test.swift:1:8:1:17 | Foundation | Foundation |
+| test.swift:5:9:5:13 | items | items |
+| test.swift:7:19:7:21 | add | add |
+| test.swift:7:23:7:23 | _ | _ |
+| test.swift:7:25:7:28 | item | item |
+| test.swift:8:9:8:13 | items | items |
+| test.swift:8:15:8:20 | append | append |
+| test.swift:8:22:8:25 | item | item |
+| test.swift:11:10:11:17 | contains | contains |
+| test.swift:11:19:11:19 | _ | _ |
+| test.swift:11:21:11:24 | item | item |
+| test.swift:12:16:12:20 | items | items |
+| test.swift:12:22:12:29 | contains | contains |
+| test.swift:12:31:12:34 | item | item |
+| test.swift:19:9:19:13 | count | count |
+| test.swift:20:10:20:13 | item | item |
+| test.swift:20:15:20:16 | at | at |
+| test.swift:20:18:20:22 | index | index |
+| test.swift:24:6:24:10 | merge | merge |
+| test.swift:24:27:24:27 | _ | _ |
+| test.swift:24:29:24:33 | first | first |
+| test.swift:24:39:24:39 | _ | _ |
+| test.swift:24:41:24:46 | second | second |
+| test.swift:24:73:24:73 | T | T |
+| test.swift:24:75:24:81 | Element | Element |
+| test.swift:25:9:25:14 | result | result |
+| test.swift:25:18:25:22 | Array | Array |
+| test.swift:25:24:25:28 | first | first |
+| test.swift:26:9:26:12 | item | item |
+| test.swift:26:17:26:22 | second | second |
+| test.swift:27:13:27:18 | result | result |
+| test.swift:27:20:27:27 | contains | contains |
+| test.swift:27:29:27:32 | item | item |
+| test.swift:28:13:28:18 | result | result |
+| test.swift:28:20:28:25 | append | append |
+| test.swift:28:27:28:30 | item | item |
+| test.swift:31:12:31:17 | result | result |
+| test.swift:37:17:37:20 | data | data |
+| test.swift:39:9:39:13 | count | count |
+| test.swift:40:16:40:19 | data | data |
+| test.swift:40:21:40:25 | count | count |
+| test.swift:43:9:43:15 | isEmpty | isEmpty |
+| test.swift:44:9:44:12 | data | data |
+| test.swift:44:14:44:20 | isEmpty | isEmpty |
+| test.swift:47:10:47:13 | item | item |
+| test.swift:47:15:47:16 | at | at |
+| test.swift:47:18:47:22 | index | index |
+| test.swift:48:15:48:19 | index | index |
+| test.swift:48:29:48:33 | index | index |
+| test.swift:48:37:48:40 | data | data |
+| test.swift:48:42:48:46 | count | count |
+| test.swift:49:16:49:19 | data | data |
+| test.swift:49:21:49:25 | index | index |
+| test.swift:52:10:52:12 | add | add |
+| test.swift:52:14:52:14 | _ | _ |
+| test.swift:52:16:52:19 | item | item |
+| test.swift:53:9:53:12 | data | data |
+| test.swift:53:14:53:19 | append | append |
+| test.swift:53:21:53:24 | item | item |
+| test.swift:59:10:59:16 | success | success |
+| test.swift:60:10:60:16 | failure | failure |
+| test.swift:62:10:62:12 | map | map |
+| test.swift:62:17:62:17 | _ | _ |
+| test.swift:62:19:62:27 | transform | transform |
+| test.swift:64:15:64:21 | success | success |
+| test.swift:64:27:64:31 | value | value |
+| test.swift:65:21:65:27 | success | success |
+| test.swift:65:29:65:37 | transform | transform |
+| test.swift:65:39:65:43 | value | value |
+| test.swift:66:15:66:21 | failure | failure |
+| test.swift:66:27:66:31 | error | error |
+| test.swift:67:21:67:27 | failure | failure |
+| test.swift:67:29:67:33 | error | error |
+| test.swift:73:23:73:29 | Element | Element |
+| test.swift:74:10:74:17 | isSorted | isSorted |
+| test.swift:75:13:75:13 | i | i |
+| test.swift:75:23:75:31 | blah | blah |
+| test.swift:76:21:76:21 | i | i |
+| test.swift:76:31:76:35 | blah | blah |
+| test.swift:85:6:85:12 | combine | combine |
+| test.swift:85:17:85:17 | _ | _ |
+| test.swift:85:19:85:24 | values | values |
+| test.swift:85:32:85:40 | transform | transform |
+| test.swift:86:12:86:17 | values | values |
+| test.swift:86:19:86:25 | isEmpty | isEmpty |
+| test.swift:87:12:87:17 | values | values |
+| test.swift:87:19:87:27 | dropFirst | dropFirst |
+| test.swift:87:31:87:36 | reduce | reduce |
+| test.swift:87:38:87:43 | values | values |
+| test.swift:87:49:87:57 | transform | transform |
+func
+| test.swift:7:5:9:5 | FunctionDeclaration |
+| test.swift:11:5:13:5 | FunctionDeclaration |
+| test.swift:24:1:32:1 | FunctionDeclaration |
+| test.swift:47:5:50:5 | FunctionDeclaration |
+| test.swift:52:5:54:5 | FunctionDeclaration |
+| test.swift:62:5:69:5 | FunctionDeclaration |
+| test.swift:74:5:81:5 | FunctionDeclaration |
+| test.swift:85:1:88:1 | FunctionDeclaration |
+add
--- a/unified/ql/test/library-tests/BasicTest/test.ql
+++ b/unified/ql/test/library-tests/BasicTest/test.ql
@@ -0,0 +1,9 @@
+import codeql.unified.Ast
+
+query predicate identifier(Swift::SimpleIdentifier node, string name) { name = node.getValue() }
+
+query predicate func(Swift::FunctionDeclaration node) { any() }
+
+query predicate add(Swift::AdditiveExpression node, Swift::AstNode lhs, Swift::AstNode rhs) {
+  lhs = node.getLhs(0) and rhs = node.getRhs(0)
+}
--- a/unified/ql/test/library-tests/BasicTest/test.swift
+++ b/unified/ql/test/library-tests/BasicTest/test.swift
@@ -0,0 +1,88 @@
+import Foundation
+
+// Generic struct with type constraint
+struct Container<T: Equatable> {
+    var items: [T] = []
+
+    mutating func add(_ item: T) {
+        items.append(item)
+    }
+
+    func contains(_ item: T) -> Bool {
+        return items.contains(item)
+    }
+}
+
+// Protocol with associated type
+protocol DataSource {
+    associatedtype Element
+    var count: Int { get }
+    func item(at index: Int) -> Element?
+}
+
+// Generic function with where clause
+func merge<T: Collection>(_ first: T, _ second: T) -> [T.Element] where T.Element: Equatable {
+    var result = Array(first)
+    for item in second {
+        if !result.contains(item) {
+            result.append(item)
+        }
+    }
+    return result
+}
+
+// Class with inheritance and computed properties
+class DataManager<T>: DataSource {
+    typealias Element = T
+    private var data: [T] = []
+
+    var count: Int {
+        return data.count
+    }
+
+    var isEmpty: Bool {
+        data.isEmpty
+    }
+
+    func item(at index: Int) -> T? {
+        guard index >= 0 && index < data.count else { return nil }
+        return data[index]
+    }
+
+    func add(_ item: T) {
+        data.append(item)
+    }
+}
+
+// Enum with associated values
+enum Result<Success, Failure: Error> {
+    case success(Success)
+    case failure(Failure)
+
+    func map<U>(_ transform: (Success) -> U) -> Result<U, Failure> {
+        switch self {
+        case .success(let value):
+            return .success(transform(value))
+        case .failure(let error):
+            return .failure(error)
+        }
+    }
+}
+
+// Extension with generic constraints
+extension Array where Element: Comparable {
+    func isSorted() -> Bool {
+        for i in 0..<(count - 1) {
+            if self[i] > self[i + 1] {
+                return false
+            }
+        }
+        return true
+    }
+}
+
+// Higher-order function
+func combine<T>(_ values: [T], transform: (T, T) -> T) -> T? {
+    guard !values.isEmpty else { return nil }
+    return values.dropFirst().reduce(values[0], transform)
+}
--- a/unified/ql/test/qlpack.lock.yml
+++ b/unified/ql/test/qlpack.lock.yml
@@ -0,0 +1,4 @@
+---
+dependencies: {}
+compiled: false
+lockVersion: 1.0.0
--- a/unified/ql/test/qlpack.yml
+++ b/unified/ql/test/qlpack.yml
@@ -0,0 +1,8 @@
+name: codeql/unified-tests
+groups: [unified, test]
+dependencies:
+  codeql/unified-queries: ${workspace}
+  codeql/unified-all: ${workspace}
+extractor: unified
+tests: .
+warnOnImplicitThis: true
--- a/unified/scripts/create-extractor-pack.sh
+++ b/unified/scripts/create-extractor-pack.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+set -eux
+if [[ "$OSTYPE" == "linux-gnu"* ]]; then
+  platform="linux64"
+elif [[ "$OSTYPE" == "darwin"* ]]; then
+  platform="osx64"
+else
+  echo "Unknown OS"
+  exit 1
+fi
+cd "$(dirname "$0")/.."
+
+(cd extractor && cargo build --release)
+
+# we are in a cargo workspace rooted at the git checkout
+BIN_DIR=../target/release
+"$BIN_DIR/codeql-extractor-unified" generate --dbscheme ql/lib/unified.dbscheme --library ql/lib/codeql/unified/Ast.qll
+
+codeql query format -i ql/lib/codeql/unified/Ast.qll
+
+rm -rf extractor-pack
+mkdir -p extractor-pack
+cp -r codeql-extractor.yml tools ql/lib/unified.dbscheme ql/lib/unified.dbscheme.stats extractor-pack/
+mkdir -p extractor-pack/tools/${platform}
+cp "$BIN_DIR/codeql-extractor-unified" extractor-pack/tools/${platform}/extractor
--- a/unified/tools/BUILD.bazel
+++ b/unified/tools/BUILD.bazel
@@ -0,0 +1,11 @@
+load("//misc/bazel:pkg.bzl", "codeql_pkg_files")
+
+codeql_pkg_files(
+    name = "tools",
+    excludes = [
+        "BUILD.bazel",
+    ],
+    exes = glob(["**/*"]),
+    prefix = "tools",
+    visibility = ["//unified:__pkg__"],
+)
--- a/unified/tools/autobuild.cmd
+++ b/unified/tools/autobuild.cmd
@@ -0,0 +1,5 @@
+@echo off
+
+type NUL && "%CODEQL_EXTRACTOR_UNIFIED_ROOT%\tools\%CODEQL_PLATFORM%\extractor" autobuild
+
+exit /b %ERRORLEVEL%
--- a/unified/tools/autobuild.sh
+++ b/unified/tools/autobuild.sh
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+exec "${CODEQL_EXTRACTOR_UNIFIED_ROOT}/tools/${CODEQL_PLATFORM}/extractor" autobuild
--- a/unified/tools/index-files.cmd
+++ b/unified/tools/index-files.cmd
@@ -0,0 +1,9 @@
+@echo off
+
+type NUL && "%CODEQL_EXTRACTOR_UNIFIED_ROOT%\tools\win64\extractor.exe" ^
+        extract ^
+        --file-list "%1" ^
+        --source-archive-dir "%CODEQL_EXTRACTOR_UNIFIED_SOURCE_ARCHIVE_DIR%" ^
+        --output-dir "%CODEQL_EXTRACTOR_UNIFIED_TRAP_DIR%"
+
+exit /b %ERRORLEVEL%
--- a/unified/tools/index-files.sh
+++ b/unified/tools/index-files.sh
@@ -0,0 +1,9 @@
+#!/bin/sh
+
+set -eu
+
+exec "${CODEQL_EXTRACTOR_UNIFIED_ROOT}/tools/${CODEQL_PLATFORM}/extractor" \
+        extract \
+        --file-list "$1" \
+        --source-archive-dir "$CODEQL_EXTRACTOR_UNIFIED_SOURCE_ARCHIVE_DIR" \
+        --output-dir "$CODEQL_EXTRACTOR_UNIFIED_TRAP_DIR"
--- a/unified/tools/qltest.cmd
+++ b/unified/tools/qltest.cmd
@@ -0,0 +1,14 @@
+@echo off
+
+type NUL && "%CODEQL_DIST%\codeql.exe" database index-files ^
+    --prune=**/*.testproj ^
+    --include-extension=.swift ^
+    --include-extension=.swiftinterface ^
+    --size-limit=5m ^
+    --language=unified ^
+    --working-dir=. ^
+    "%CODEQL_EXTRACTOR_UNIFIED_WIP_DATABASE%"
+
+IF %ERRORLEVEL% NEQ 0 exit /b %ERRORLEVEL%
+
+exit /b %ERRORLEVEL%
--- a/unified/tools/qltest.sh
+++ b/unified/tools/qltest.sh
@@ -0,0 +1,12 @@
+#!/bin/sh
+
+set -eu
+
+"${CODEQL_DIST}/codeql" database index-files \
+    --prune="**/*.testproj" \
+    --include-extension=.swift \
+    --include-extension=.swiftinterface \
+    --size-limit=5m \
+    --language=unified \
+    --working-dir=.\
+    "$CODEQL_EXTRACTOR_UNIFIED_WIP_DATABASE"