Merge pull request #15656 from github/criemen/ruby-bazel

Ruby: Start building the language pack using bazel.

.gitattributes

@@ -74,3 +74,7 @@ javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/auto
 
 # Auto-generated modeling for Python
 python/ql/lib/semmle/python/frameworks/data/internal/subclass-capture/*.yml linguist-generated=true
+
+# auto-generated bazel lock file
+ruby/extractor/cargo-bazel-lock.json linguist-generated=true
+ruby/extractor/cargo-bazel-lock.json -merge

.github/workflows/ruby-build.yml

@@ -51,9 +51,11 @@ jobs:
         run: |
           brew install gnu-tar
           echo "/usr/local/opt/gnu-tar/libexec/gnubin" >> $GITHUB_PATH
-      - name: Install cargo-cross
-        if: runner.os == 'Linux'
-        run: cargo install cross --version 0.2.5
+      - name: Prepare Windows
+        if: runner.os == 'Windows'
+        shell: powershell
+        run: |
+          git config --global core.longpaths true
       - uses: ./.github/actions/os-version
         id: os_version
       - name: Cache entire extractor
@@ -82,16 +84,8 @@ jobs:
       - name: Run tests
         if: steps.cache-extractor.outputs.cache-hit != 'true'
         run: cd extractor && cargo test --verbose
-      # On linux, build the extractor via cross in a centos7 container.
-      # This ensures we don't depend on glibc > 2.17.
-      - name: Release build (linux)
-        if: steps.cache-extractor.outputs.cache-hit != 'true' && runner.os == 'Linux'
-        run: |
-          cd extractor
-          cross build --release
-          mv target/x86_64-unknown-linux-gnu/release/codeql-extractor-ruby target/release/
-      - name: Release build (windows and macos)
-        if: steps.cache-extractor.outputs.cache-hit != 'true' && runner.os != 'Linux'
+      - name: Release build
+        if: steps.cache-extractor.outputs.cache-hit != 'true'
         run: cd extractor && cargo build --release
       - name: Generate dbscheme
         if: ${{ matrix.os == 'ubuntu-latest' && steps.cache-extractor.outputs.cache-hit != 'true'}}
@@ -123,7 +117,7 @@ jobs:
       - name: Cache compilation cache
         id: query-cache
         uses: ./.github/actions/cache-query-compilation
-        with: 
+        with:
           key: ruby-build
       - name: Build Query Pack
         run: |
@@ -235,54 +229,3 @@ jobs:
         shell: bash
         run: |
           codeql database analyze --search-path "${{ runner.temp }}/ruby-bundle" --format=sarifv2.1.0 --output=out.sarif ../database ruby-code-scanning.qls
-
-  # This is a copy of the 'test' job that runs in a centos7 container.
-  # This tests that the extractor works correctly on systems with an old glibc.
-  test-centos7:
-    defaults:
-      run:
-        working-directory: ${{ github.workspace }}
-    strategy:
-      fail-fast: false
-    runs-on: ubuntu-latest
-    container:
-      image: centos:centos7
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-    needs: [package]
-    steps:
-      - name: Install gh cli
-        run: |
-          yum-config-manager --add-repo https://cli.github.com/packages/rpm/gh-cli.repo
-          # fetch-codeql requires unzip and jq
-          # jq is available in epel-release (https://docs.fedoraproject.org/en-US/epel/)
-          yum install -y gh unzip epel-release
-          yum install -y jq
-      - uses: actions/checkout@v3
-      - name: Fetch CodeQL
-        uses: ./.github/actions/fetch-codeql
-
-      # Due to a bug in Actions, we can't use runner.temp in the run blocks here.
-      # https://github.com/actions/runner/issues/2185
-
-      - name: Download Ruby bundle
-        uses: actions/download-artifact@v3
-        with:
-          name: codeql-ruby-bundle
-          path: ${{ runner.temp }}
-      - name: Unzip Ruby bundle
-        shell: bash
-        run: unzip -q -d "$RUNNER_TEMP"/ruby-bundle "$RUNNER_TEMP"/codeql-ruby-bundle.zip
-
-      - name: Run QL test
-        shell: bash
-        run: |
-          codeql test run --search-path "$RUNNER_TEMP"/ruby-bundle --additional-packs "$RUNNER_TEMP"/ruby-bundle ruby/ql/test/library-tests/ast/constants/
-      - name: Create database
-        shell: bash
-        run: |
-          codeql database create --search-path "$RUNNER_TEMP"/ruby-bundle --language ruby --source-root ruby/ql/test/library-tests/ast/constants/ ../database
-      - name: Analyze database
-        shell: bash
-        run: |
-          codeql database analyze --search-path "$RUNNER_TEMP"/ruby-bundle --format=sarifv2.1.0 --output=out.sarif ../database ruby-code-scanning.qls

ruby/BUILD.bazel

@@ -0,0 +1,60 @@
+load("@rules_pkg//pkg:mappings.bzl", "pkg_filegroup", "pkg_files")
+load("@semmle_code//:dist.bzl", "dist", "pack_zip")
+load("//:defs.bzl", "codeql_platform")
+
+package(default_visibility = ["//visibility:public"])
+
+alias(
+    name = "dbscheme",
+    actual = "//ruby/ql/lib:dbscheme",
+)
+
+alias(
+    name = "dbscheme-stats",
+    actual = "//ruby/ql/lib:dbscheme-stats",
+)
+
+pkg_files(
+    name = "dbscheme-group",
+    srcs = [
+        ":dbscheme",
+        ":dbscheme-stats",
+    ],
+    strip_prefix = None,
+)
+
+pkg_filegroup(
+    name = "db-files",
+    srcs = [
+        ":dbscheme-group",
+        "//ruby/downgrades",
+    ],
+)
+
+pkg_files(
+    name = "codeql-extractor-yml",
+    srcs = ["codeql-extractor.yml"],
+    strip_prefix = None,
+)
+
+dist(
+    name = "extractor-generic",
+    srcs = [
+        ":codeql-extractor-yml",
+        ":dbscheme-group",
+        "//ruby/downgrades",
+        "//ruby/tools",
+    ],
+    prefix = "ruby",
+    visibility = ["//visibility:public"],
+)
+
+pack_zip(
+    name = "extractor-arch",
+    srcs = [
+        "//ruby/extractor",
+    ],
+    package_file_name = "extractor-" + codeql_platform + ".zip",
+    prefix = "ruby/tools/" + codeql_platform,
+    visibility = ["//visibility:public"],
+)

ruby/downgrades/BUILD.bazel

@@ -0,0 +1,12 @@
+load("@rules_pkg//:mappings.bzl", "pkg_files", "strip_prefix")
+
+pkg_files(
+    name = "downgrades",
+    srcs = glob(
+        ["**"],
+        exclude = ["BUILD.bazel"],
+    ),
+    prefix = "downgrades",
+    strip_prefix = strip_prefix.from_pkg(),
+    visibility = ["//ruby:__pkg__"],
+)

ruby/extractor/BUILD.bazel

@@ -0,0 +1,15 @@
+load("@ruby_deps//:defs.bzl", "aliases", "all_crate_deps")
+load("@semmle_code//:common.bzl", "codeql_rust_binary")
+
+codeql_rust_binary(
+    name = "extractor",
+    srcs = glob(["src/*.rs"]),
+    aliases = aliases(),
+    proc_macro_deps = all_crate_deps(
+        proc_macro = True,
+    ),
+    visibility = ["//visibility:public"],
+    deps = all_crate_deps(
+        normal = True,
+    ),
+)

ruby/extractor/Cargo.toml

@@ -5,8 +5,15 @@ version = "0.1.0"
 authors = ["GitHub"]
 edition = "2018"
 
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
+# When changing/updating these, the `cargo-bazel-lock.json` file has to be regenerated.
+# Run `CARGO_BAZEL_REPIN=true CARGO_BAZEL_REPIN_ONLY=ruby_deps ./build --bazel sync --only=ruby_deps`
+# in the `semmle-code` repository to do so.
+# For more information, check out the documentation at
+# https://bazelbuild.github.io/rules_rust/crate_universe.html#repinning--updating-dependencies
+# In the future, the hope is to move this handling of the dependencies entirely into the `codeql` repository,
+# but that depends on `rules_rust` being fully compatible with bzlmod, which they aren't yet
+# (c.f. https://github.com/bazelbuild/rules_rust/issues/2452).
+# Warning: The process takes >5min on my M1 mac, so do wait for a while.
 [dependencies]
 tree-sitter = "0.20"
 tree-sitter-embedded-template = { git = "https://github.com/tree-sitter/tree-sitter-embedded-template.git", rev = "203f7bd3c1bbfbd98fc19add4b8fcb213c059205" }
@@ -18,4 +25,13 @@ rayon = "1.5.0"
 regex = "1.7.1"
 encoding = "0.2"
 lazy_static = "1.4.0"
-codeql-extractor = { path = "../../shared/tree-sitter-extractor" }
+# Ideally, we'd like to pull this in via a relative path.
+# However, our bazel/rust tooling chokes on this, c.f. https://github.com/bazelbuild/rules_rust/issues/1525
+# Therefore, to break that dependency, we depend on it via a git dependency instead.
+# We should change this back to a path dependency once this issue is fixed.
+# We can't depend on this without a rev/branch specification, as the rules_rust code assumes the default branch
+# is called `master`, and if we pull this in with `branch=main`, then `cargo` works (and pins this at th current git SHA
+# of lock-file update time, but `rules_rust` pins generates a bazel rule that unconditionally downloads `main`, which
+# breaks build hermeticity. So, rev-pinning it is.
+# See also https://github.com/bazelbuild/rules_rust/issues/2502.
+codeql-extractor = { git = "https://github.com/github/codeql.git", rev = "514a92d5bd1e24e4b7367d64430762ffd1ffbe7f" }

ruby/extractor/Cross.toml

@@ -1,8 +0,0 @@
-[target.x86_64-unknown-linux-gnu]
-image = "centos/devtoolset-7-toolchain-centos7"
-
-[build.env]
-# Provide the path to the shared extractor
-# Cross mounts this directory as a volume, so builds inside the docker container
-# can see it.
-volumes = ["__CODEQL-EXTRACTOR=../../shared/tree-sitter-extractor"]

ruby/ql/lib/BUILD.bazel

@@ -0,0 +1,13 @@
+load("@rules_pkg//:mappings.bzl", "pkg_files")
+
+package(default_visibility = ["//ruby:__pkg__"])
+
+pkg_files(
+    name = "dbscheme",
+    srcs = ["ruby.dbscheme"],
+)
+
+pkg_files(
+    name = "dbscheme-stats",
+    srcs = ["ruby.dbscheme.stats"],
+)

ruby/tools/BUILD.bazel

@@ -0,0 +1,11 @@
+load("@semmle_code//:dist.bzl", "pack_zip")
+
+pack_zip(
+    name = "tools",
+    srcs = glob(["**/*"]),
+    excludes = [
+        "BUILD.bazel",
+    ],
+    prefix = "tools",
+    visibility = ["//visibility:public"],
+)