hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2025-12-16 16:53:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #19613 from github/redsun82/rust-compression

Browse Source 
Ruby, Rust: add zstd compression option (and fix compression in Rust)
This commit is contained in:
Paolo Tranquilli
2025-05-28 17:20:57 +02:00
committed by GitHub
parent 7d1a38f42a c8f5e26200
commit 80a5dde240
21 changed files with 895 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
Cargo.lock generated
View File

@@ -242,6 +242,8 @@ version = "1.2.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a012a0df96dd6d06ba9a1b29d6402d1a5d77c6befd2566afdc26e10603dc93d7"
dependencies = [
"jobserver",
"libc",
"shlex",
]
@@ -390,6 +392,7 @@ dependencies = [
"tree-sitter",
"tree-sitter-json",
"tree-sitter-ql",
"zstd",
]
[[package]]
@@ -983,6 +986,15 @@ version = "1.0.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c"
[[package]]
name = "jobserver"
version = "0.1.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0"
dependencies = [
"libc",
]
[[package]]
name = "jod-thread"
version = "0.1.2"
@@ -1334,6 +1346,12 @@ version = "0.2.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b"
[[package]]
name = "pkg-config"
version = "0.3.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c"
[[package]]
name = "portable-atomic"
version = "1.11.0"
@@ -3027,3 +3045,31 @@ dependencies = [
"quote",
"syn",
]
[[package]]
name = "zstd"
version = "0.13.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a"
dependencies = [
"zstd-safe",
]
[[package]]
name = "zstd-safe"
version = "7.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d"
dependencies = [
"zstd-sys",
]
[[package]]
name = "zstd-sys"
version = "2.0.15+zstd.1.5.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eb81183ddd97d0c74cedf1d50d85c8d08c1b8b68ee863bdee9e706eedba1a237"
dependencies = [
"cc",
"pkg-config",
]

1
MODULE.bazel
View File

@@ -124,6 +124,7 @@ use_repo(
"vendor_ts__tree-sitter-ruby-0.23.1",
"vendor_ts__triomphe-0.1.14",
"vendor_ts__ungrammar-1.16.1",
"vendor_ts__zstd-0.13.3",
)
http_archive = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")

12
misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.bazel generated vendored
View File

@@ -672,3 +672,15 @@ alias(
actual = "@vendor_ts__ungrammar-1.16.1//:ungrammar",
tags = ["manual"],
)
alias(
name = "zstd-0.13.3",
actual = "@vendor_ts__zstd-0.13.3//:zstd",
tags = ["manual"],
)
alias(
name = "zstd",
actual = "@vendor_ts__zstd-0.13.3//:zstd",
tags = ["manual"],
)

41
misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.cc-1.2.7.bazel generated vendored
View File

@@ -28,6 +28,9 @@ rust_library(
"WORKSPACE.bazel",
],
),
crate_features = [
"parallel",
],
crate_root = "src/lib.rs",
edition = "2018",
rustc_flags = [
@@ -81,6 +84,42 @@ rust_library(
}),
version = "1.2.7",
deps = [
"@vendor_ts__jobserver-0.1.32//:jobserver",
"@vendor_ts__shlex-1.3.0//:shlex",
],
] + select({
"@rules_rust//rust/platform:aarch64-apple-darwin": [
"@vendor_ts__libc-0.2.171//:libc", # aarch64-apple-darwin
],
"@rules_rust//rust/platform:aarch64-unknown-linux-gnu": [
"@vendor_ts__libc-0.2.171//:libc", # aarch64-unknown-linux-gnu
],
"@rules_rust//rust/platform:aarch64-unknown-nixos-gnu": [
"@vendor_ts__libc-0.2.171//:libc", # aarch64-unknown-linux-gnu, aarch64-unknown-nixos-gnu
],
"@rules_rust//rust/platform:arm-unknown-linux-gnueabi": [
"@vendor_ts__libc-0.2.171//:libc", # arm-unknown-linux-gnueabi
],
"@rules_rust//rust/platform:i686-unknown-linux-gnu": [
"@vendor_ts__libc-0.2.171//:libc", # i686-unknown-linux-gnu
],
"@rules_rust//rust/platform:powerpc-unknown-linux-gnu": [
"@vendor_ts__libc-0.2.171//:libc", # powerpc-unknown-linux-gnu
],
"@rules_rust//rust/platform:s390x-unknown-linux-gnu": [
"@vendor_ts__libc-0.2.171//:libc", # s390x-unknown-linux-gnu
],
"@rules_rust//rust/platform:x86_64-apple-darwin": [
"@vendor_ts__libc-0.2.171//:libc", # x86_64-apple-darwin
],
"@rules_rust//rust/platform:x86_64-unknown-freebsd": [
"@vendor_ts__libc-0.2.171//:libc", # x86_64-unknown-freebsd
],
"@rules_rust//rust/platform:x86_64-unknown-linux-gnu": [
"@vendor_ts__libc-0.2.171//:libc", # x86_64-unknown-linux-gnu
],
"@rules_rust//rust/platform:x86_64-unknown-nixos-gnu": [
"@vendor_ts__libc-0.2.171//:libc", # x86_64-unknown-linux-gnu, x86_64-unknown-nixos-gnu
],
"//conditions:default": [],
}),
)

158
misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.jobserver-0.1.32.bazel generated vendored Normal file
View File

@@ -0,0 +1,158 @@
###############################################################################
# @generated
# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To
# regenerate this file, run the following:
#
# bazel run @@//misc/bazel/3rdparty:vendor_tree_sitter_extractors
###############################################################################
load("@rules_rust//rust:defs.bzl", "rust_library")
package(default_visibility = ["//visibility:public"])
rust_library(
name = "jobserver",
srcs = glob(
include = ["**/*.rs"],
allow_empty = True,
),
compile_data = glob(
include = ["**"],
allow_empty = True,
exclude = [
"**/* *",
".tmp_git_root/**/*",
"BUILD",
"BUILD.bazel",
"WORKSPACE",
"WORKSPACE.bazel",
],
),
crate_root = "src/lib.rs",
edition = "2021",
rustc_flags = [
"--cap-lints=allow",
],
tags = [
"cargo-bazel",
"crate-name=jobserver",
"manual",
"noclippy",
"norustfmt",
],
target_compatible_with = select({
"@rules_rust//rust/platform:aarch64-apple-darwin": [],
"@rules_rust//rust/platform:aarch64-apple-ios": [],
"@rules_rust//rust/platform:aarch64-apple-ios-sim": [],
"@rules_rust//rust/platform:aarch64-linux-android": [],
"@rules_rust//rust/platform:aarch64-pc-windows-msvc": [],
"@rules_rust//rust/platform:aarch64-unknown-fuchsia": [],
"@rules_rust//rust/platform:aarch64-unknown-linux-gnu": [],
"@rules_rust//rust/platform:aarch64-unknown-nixos-gnu": [],
"@rules_rust//rust/platform:aarch64-unknown-nto-qnx710": [],
"@rules_rust//rust/platform:aarch64-unknown-uefi": [],
"@rules_rust//rust/platform:arm-unknown-linux-gnueabi": [],
"@rules_rust//rust/platform:armv7-linux-androideabi": [],
"@rules_rust//rust/platform:armv7-unknown-linux-gnueabi": [],
"@rules_rust//rust/platform:i686-apple-darwin": [],
"@rules_rust//rust/platform:i686-linux-android": [],
"@rules_rust//rust/platform:i686-pc-windows-msvc": [],
"@rules_rust//rust/platform:i686-unknown-freebsd": [],
"@rules_rust//rust/platform:i686-unknown-linux-gnu": [],
"@rules_rust//rust/platform:powerpc-unknown-linux-gnu": [],
"@rules_rust//rust/platform:riscv32imc-unknown-none-elf": [],
"@rules_rust//rust/platform:riscv64gc-unknown-none-elf": [],
"@rules_rust//rust/platform:s390x-unknown-linux-gnu": [],
"@rules_rust//rust/platform:thumbv7em-none-eabi": [],
"@rules_rust//rust/platform:thumbv8m.main-none-eabi": [],
"@rules_rust//rust/platform:wasm32-unknown-unknown": [],
"@rules_rust//rust/platform:wasm32-wasip1": [],
"@rules_rust//rust/platform:x86_64-apple-darwin": [],
"@rules_rust//rust/platform:x86_64-apple-ios": [],
"@rules_rust//rust/platform:x86_64-linux-android": [],
"@rules_rust//rust/platform:x86_64-pc-windows-msvc": [],
"@rules_rust//rust/platform:x86_64-unknown-freebsd": [],
"@rules_rust//rust/platform:x86_64-unknown-fuchsia": [],
"@rules_rust//rust/platform:x86_64-unknown-linux-gnu": [],
"@rules_rust//rust/platform:x86_64-unknown-nixos-gnu": [],
"@rules_rust//rust/platform:x86_64-unknown-none": [],
"@rules_rust//rust/platform:x86_64-unknown-uefi": [],
"//conditions:default": ["@platforms//:incompatible"],
}),
version = "0.1.32",
deps = select({
"@rules_rust//rust/platform:aarch64-apple-darwin": [
"@vendor_ts__libc-0.2.171//:libc", # cfg(unix)
],
"@rules_rust//rust/platform:aarch64-apple-ios": [
"@vendor_ts__libc-0.2.171//:libc", # cfg(unix)
],
"@rules_rust//rust/platform:aarch64-apple-ios-sim": [
"@vendor_ts__libc-0.2.171//:libc", # cfg(unix)
],
"@rules_rust//rust/platform:aarch64-linux-android": [
"@vendor_ts__libc-0.2.171//:libc", # cfg(unix)
],
"@rules_rust//rust/platform:aarch64-unknown-fuchsia": [
"@vendor_ts__libc-0.2.171//:libc", # cfg(unix)
],
"@rules_rust//rust/platform:aarch64-unknown-linux-gnu": [
"@vendor_ts__libc-0.2.171//:libc", # cfg(unix)
],
"@rules_rust//rust/platform:aarch64-unknown-nixos-gnu": [
"@vendor_ts__libc-0.2.171//:libc", # cfg(unix)
],
"@rules_rust//rust/platform:aarch64-unknown-nto-qnx710": [
"@vendor_ts__libc-0.2.171//:libc", # cfg(unix)
],
"@rules_rust//rust/platform:arm-unknown-linux-gnueabi": [
"@vendor_ts__libc-0.2.171//:libc", # cfg(unix)
],
"@rules_rust//rust/platform:armv7-linux-androideabi": [
"@vendor_ts__libc-0.2.171//:libc", # cfg(unix)
],
"@rules_rust//rust/platform:armv7-unknown-linux-gnueabi": [
"@vendor_ts__libc-0.2.171//:libc", # cfg(unix)
],
"@rules_rust//rust/platform:i686-apple-darwin": [
"@vendor_ts__libc-0.2.171//:libc", # cfg(unix)
],
"@rules_rust//rust/platform:i686-linux-android": [
"@vendor_ts__libc-0.2.171//:libc", # cfg(unix)
],
"@rules_rust//rust/platform:i686-unknown-freebsd": [
"@vendor_ts__libc-0.2.171//:libc", # cfg(unix)
],
"@rules_rust//rust/platform:i686-unknown-linux-gnu": [
"@vendor_ts__libc-0.2.171//:libc", # cfg(unix)
],
"@rules_rust//rust/platform:powerpc-unknown-linux-gnu": [
"@vendor_ts__libc-0.2.171//:libc", # cfg(unix)
],
"@rules_rust//rust/platform:s390x-unknown-linux-gnu": [
"@vendor_ts__libc-0.2.171//:libc", # cfg(unix)
],
"@rules_rust//rust/platform:x86_64-apple-darwin": [
"@vendor_ts__libc-0.2.171//:libc", # cfg(unix)
],
"@rules_rust//rust/platform:x86_64-apple-ios": [
"@vendor_ts__libc-0.2.171//:libc", # cfg(unix)
],
"@rules_rust//rust/platform:x86_64-linux-android": [
"@vendor_ts__libc-0.2.171//:libc", # cfg(unix)
],
"@rules_rust//rust/platform:x86_64-unknown-freebsd": [
"@vendor_ts__libc-0.2.171//:libc", # cfg(unix)
],
"@rules_rust//rust/platform:x86_64-unknown-fuchsia": [
"@vendor_ts__libc-0.2.171//:libc", # cfg(unix)
],
"@rules_rust//rust/platform:x86_64-unknown-linux-gnu": [
"@vendor_ts__libc-0.2.171//:libc", # cfg(unix)
],
"@rules_rust//rust/platform:x86_64-unknown-nixos-gnu": [
"@vendor_ts__libc-0.2.171//:libc", # cfg(unix)
],
"//conditions:default": [],
}),
)

83
misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.pkg-config-0.3.32.bazel generated vendored Normal file
View File

@@ -0,0 +1,83 @@
###############################################################################
# @generated
# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To
# regenerate this file, run the following:
#
# bazel run @@//misc/bazel/3rdparty:vendor_tree_sitter_extractors
###############################################################################
load("@rules_rust//rust:defs.bzl", "rust_library")
package(default_visibility = ["//visibility:public"])
rust_library(
name = "pkg_config",
srcs = glob(
include = ["**/*.rs"],
allow_empty = True,
),
compile_data = glob(
include = ["**"],
allow_empty = True,
exclude = [
"**/* *",
".tmp_git_root/**/*",
"BUILD",
"BUILD.bazel",
"WORKSPACE",
"WORKSPACE.bazel",
],
),
crate_root = "src/lib.rs",
edition = "2018",
rustc_flags = [
"--cap-lints=allow",
],
tags = [
"cargo-bazel",
"crate-name=pkg-config",
"manual",
"noclippy",
"norustfmt",
],
target_compatible_with = select({
"@rules_rust//rust/platform:aarch64-apple-darwin": [],
"@rules_rust//rust/platform:aarch64-apple-ios": [],
"@rules_rust//rust/platform:aarch64-apple-ios-sim": [],
"@rules_rust//rust/platform:aarch64-linux-android": [],
"@rules_rust//rust/platform:aarch64-pc-windows-msvc": [],
"@rules_rust//rust/platform:aarch64-unknown-fuchsia": [],
"@rules_rust//rust/platform:aarch64-unknown-linux-gnu": [],
"@rules_rust//rust/platform:aarch64-unknown-nixos-gnu": [],
"@rules_rust//rust/platform:aarch64-unknown-nto-qnx710": [],
"@rules_rust//rust/platform:aarch64-unknown-uefi": [],
"@rules_rust//rust/platform:arm-unknown-linux-gnueabi": [],
"@rules_rust//rust/platform:armv7-linux-androideabi": [],
"@rules_rust//rust/platform:armv7-unknown-linux-gnueabi": [],
"@rules_rust//rust/platform:i686-apple-darwin": [],
"@rules_rust//rust/platform:i686-linux-android": [],
"@rules_rust//rust/platform:i686-pc-windows-msvc": [],
"@rules_rust//rust/platform:i686-unknown-freebsd": [],
"@rules_rust//rust/platform:i686-unknown-linux-gnu": [],
"@rules_rust//rust/platform:powerpc-unknown-linux-gnu": [],
"@rules_rust//rust/platform:riscv32imc-unknown-none-elf": [],
"@rules_rust//rust/platform:riscv64gc-unknown-none-elf": [],
"@rules_rust//rust/platform:s390x-unknown-linux-gnu": [],
"@rules_rust//rust/platform:thumbv7em-none-eabi": [],
"@rules_rust//rust/platform:thumbv8m.main-none-eabi": [],
"@rules_rust//rust/platform:wasm32-unknown-unknown": [],
"@rules_rust//rust/platform:wasm32-wasip1": [],
"@rules_rust//rust/platform:x86_64-apple-darwin": [],
"@rules_rust//rust/platform:x86_64-apple-ios": [],
"@rules_rust//rust/platform:x86_64-linux-android": [],
"@rules_rust//rust/platform:x86_64-pc-windows-msvc": [],
"@rules_rust//rust/platform:x86_64-unknown-freebsd": [],
"@rules_rust//rust/platform:x86_64-unknown-fuchsia": [],
"@rules_rust//rust/platform:x86_64-unknown-linux-gnu": [],
"@rules_rust//rust/platform:x86_64-unknown-nixos-gnu": [],
"@rules_rust//rust/platform:x86_64-unknown-none": [],
"@rules_rust//rust/platform:x86_64-unknown-uefi": [],
"//conditions:default": ["@platforms//:incompatible"],
}),
version = "0.3.32",
)

92
misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.zstd-0.13.3.bazel generated vendored Normal file
View File

@@ -0,0 +1,92 @@
###############################################################################
# @generated
# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To
# regenerate this file, run the following:
#
# bazel run @@//misc/bazel/3rdparty:vendor_tree_sitter_extractors
###############################################################################
load("@rules_rust//rust:defs.bzl", "rust_library")
package(default_visibility = ["//visibility:public"])
rust_library(
name = "zstd",
srcs = glob(
include = ["**/*.rs"],
allow_empty = True,
),
compile_data = glob(
include = ["**"],
allow_empty = True,
exclude = [
"**/* *",
".tmp_git_root/**/*",
"BUILD",
"BUILD.bazel",
"WORKSPACE",
"WORKSPACE.bazel",
],
),
crate_features = [
"arrays",
"default",
"legacy",
"zdict_builder",
],
crate_root = "src/lib.rs",
edition = "2018",
rustc_flags = [
"--cap-lints=allow",
],
tags = [
"cargo-bazel",
"crate-name=zstd",
"manual",
"noclippy",
"norustfmt",
],
target_compatible_with = select({
"@rules_rust//rust/platform:aarch64-apple-darwin": [],
"@rules_rust//rust/platform:aarch64-apple-ios": [],
"@rules_rust//rust/platform:aarch64-apple-ios-sim": [],
"@rules_rust//rust/platform:aarch64-linux-android": [],
"@rules_rust//rust/platform:aarch64-pc-windows-msvc": [],
"@rules_rust//rust/platform:aarch64-unknown-fuchsia": [],
"@rules_rust//rust/platform:aarch64-unknown-linux-gnu": [],
"@rules_rust//rust/platform:aarch64-unknown-nixos-gnu": [],
"@rules_rust//rust/platform:aarch64-unknown-nto-qnx710": [],
"@rules_rust//rust/platform:aarch64-unknown-uefi": [],
"@rules_rust//rust/platform:arm-unknown-linux-gnueabi": [],
"@rules_rust//rust/platform:armv7-linux-androideabi": [],
"@rules_rust//rust/platform:armv7-unknown-linux-gnueabi": [],
"@rules_rust//rust/platform:i686-apple-darwin": [],
"@rules_rust//rust/platform:i686-linux-android": [],
"@rules_rust//rust/platform:i686-pc-windows-msvc": [],
"@rules_rust//rust/platform:i686-unknown-freebsd": [],
"@rules_rust//rust/platform:i686-unknown-linux-gnu": [],
"@rules_rust//rust/platform:powerpc-unknown-linux-gnu": [],
"@rules_rust//rust/platform:riscv32imc-unknown-none-elf": [],
"@rules_rust//rust/platform:riscv64gc-unknown-none-elf": [],
"@rules_rust//rust/platform:s390x-unknown-linux-gnu": [],
"@rules_rust//rust/platform:thumbv7em-none-eabi": [],
"@rules_rust//rust/platform:thumbv8m.main-none-eabi": [],
"@rules_rust//rust/platform:wasm32-unknown-unknown": [],
"@rules_rust//rust/platform:wasm32-wasip1": [],
"@rules_rust//rust/platform:x86_64-apple-darwin": [],
"@rules_rust//rust/platform:x86_64-apple-ios": [],
"@rules_rust//rust/platform:x86_64-linux-android": [],
"@rules_rust//rust/platform:x86_64-pc-windows-msvc": [],
"@rules_rust//rust/platform:x86_64-unknown-freebsd": [],
"@rules_rust//rust/platform:x86_64-unknown-fuchsia": [],
"@rules_rust//rust/platform:x86_64-unknown-linux-gnu": [],
"@rules_rust//rust/platform:x86_64-unknown-nixos-gnu": [],
"@rules_rust//rust/platform:x86_64-unknown-none": [],
"@rules_rust//rust/platform:x86_64-unknown-uefi": [],
"//conditions:default": ["@platforms//:incompatible"],
}),
version = "0.13.3",
deps = [
"@vendor_ts__zstd-safe-7.2.4//:zstd_safe",
],
)

158
misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.zstd-safe-7.2.4.bazel generated vendored Normal file
View File

@@ -0,0 +1,158 @@
###############################################################################
# @generated
# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To
# regenerate this file, run the following:
#
# bazel run @@//misc/bazel/3rdparty:vendor_tree_sitter_extractors
###############################################################################
load("@rules_rust//cargo:defs.bzl", "cargo_build_script")
load("@rules_rust//rust:defs.bzl", "rust_library")
package(default_visibility = ["//visibility:public"])
rust_library(
name = "zstd_safe",
srcs = glob(
include = ["**/*.rs"],
allow_empty = True,
),
compile_data = glob(
include = ["**"],
allow_empty = True,
exclude = [
"**/* *",
".tmp_git_root/**/*",
"BUILD",
"BUILD.bazel",
"WORKSPACE",
"WORKSPACE.bazel",
],
),
crate_features = [
"arrays",
"legacy",
"std",
"zdict_builder",
],
crate_root = "src/lib.rs",
edition = "2018",
rustc_flags = [
"--cap-lints=allow",
],
tags = [
"cargo-bazel",
"crate-name=zstd-safe",
"manual",
"noclippy",
"norustfmt",
],
target_compatible_with = select({
"@rules_rust//rust/platform:aarch64-apple-darwin": [],
"@rules_rust//rust/platform:aarch64-apple-ios": [],
"@rules_rust//rust/platform:aarch64-apple-ios-sim": [],
"@rules_rust//rust/platform:aarch64-linux-android": [],
"@rules_rust//rust/platform:aarch64-pc-windows-msvc": [],
"@rules_rust//rust/platform:aarch64-unknown-fuchsia": [],
"@rules_rust//rust/platform:aarch64-unknown-linux-gnu": [],
"@rules_rust//rust/platform:aarch64-unknown-nixos-gnu": [],
"@rules_rust//rust/platform:aarch64-unknown-nto-qnx710": [],
"@rules_rust//rust/platform:aarch64-unknown-uefi": [],
"@rules_rust//rust/platform:arm-unknown-linux-gnueabi": [],
"@rules_rust//rust/platform:armv7-linux-androideabi": [],
"@rules_rust//rust/platform:armv7-unknown-linux-gnueabi": [],
"@rules_rust//rust/platform:i686-apple-darwin": [],
"@rules_rust//rust/platform:i686-linux-android": [],
"@rules_rust//rust/platform:i686-pc-windows-msvc": [],
"@rules_rust//rust/platform:i686-unknown-freebsd": [],
"@rules_rust//rust/platform:i686-unknown-linux-gnu": [],
"@rules_rust//rust/platform:powerpc-unknown-linux-gnu": [],
"@rules_rust//rust/platform:riscv32imc-unknown-none-elf": [],
"@rules_rust//rust/platform:riscv64gc-unknown-none-elf": [],
"@rules_rust//rust/platform:s390x-unknown-linux-gnu": [],
"@rules_rust//rust/platform:thumbv7em-none-eabi": [],
"@rules_rust//rust/platform:thumbv8m.main-none-eabi": [],
"@rules_rust//rust/platform:wasm32-unknown-unknown": [],
"@rules_rust//rust/platform:wasm32-wasip1": [],
"@rules_rust//rust/platform:x86_64-apple-darwin": [],
"@rules_rust//rust/platform:x86_64-apple-ios": [],
"@rules_rust//rust/platform:x86_64-linux-android": [],
"@rules_rust//rust/platform:x86_64-pc-windows-msvc": [],
"@rules_rust//rust/platform:x86_64-unknown-freebsd": [],
"@rules_rust//rust/platform:x86_64-unknown-fuchsia": [],
"@rules_rust//rust/platform:x86_64-unknown-linux-gnu": [],
"@rules_rust//rust/platform:x86_64-unknown-nixos-gnu": [],
"@rules_rust//rust/platform:x86_64-unknown-none": [],
"@rules_rust//rust/platform:x86_64-unknown-uefi": [],
"//conditions:default": ["@platforms//:incompatible"],
}),
version = "7.2.4",
deps = [
"@vendor_ts__zstd-safe-7.2.4//:build_script_build",
"@vendor_ts__zstd-sys-2.0.15-zstd.1.5.7//:zstd_sys",
],
)
cargo_build_script(
name = "_bs",
srcs = glob(
include = ["**/*.rs"],
allow_empty = True,
),
compile_data = glob(
include = ["**"],
allow_empty = True,
exclude = [
"**/* *",
"**/*.rs",
".tmp_git_root/**/*",
"BUILD",
"BUILD.bazel",
"WORKSPACE",
"WORKSPACE.bazel",
],
),
crate_features = [
"arrays",
"legacy",
"std",
"zdict_builder",
],
crate_name = "build_script_build",
crate_root = "build.rs",
data = glob(
include = ["**"],
allow_empty = True,
exclude = [
"**/* *",
".tmp_git_root/**/*",
"BUILD",
"BUILD.bazel",
"WORKSPACE",
"WORKSPACE.bazel",
],
),
edition = "2018",
link_deps = [
"@vendor_ts__zstd-sys-2.0.15-zstd.1.5.7//:zstd_sys",
],
pkg_name = "zstd-safe",
rustc_flags = [
"--cap-lints=allow",
],
tags = [
"cargo-bazel",
"crate-name=zstd-safe",
"manual",
"noclippy",
"norustfmt",
],
version = "7.2.4",
visibility = ["//visibility:private"],
)
alias(
name = "build_script_build",
actual = ":_bs",
tags = ["manual"],
)

157
misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.zstd-sys-2.0.15+zstd.1.5.7.bazel generated vendored Normal file
View File

@@ -0,0 +1,157 @@
###############################################################################
# @generated
# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To
# regenerate this file, run the following:
#
# bazel run @@//misc/bazel/3rdparty:vendor_tree_sitter_extractors
###############################################################################
load("@rules_rust//cargo:defs.bzl", "cargo_build_script")
load("@rules_rust//rust:defs.bzl", "rust_library")
package(default_visibility = ["//visibility:public"])
rust_library(
name = "zstd_sys",
srcs = glob(
include = ["**/*.rs"],
allow_empty = True,
),
compile_data = glob(
include = ["**"],
allow_empty = True,
exclude = [
"**/* *",
".tmp_git_root/**/*",
"BUILD",
"BUILD.bazel",
"WORKSPACE",
"WORKSPACE.bazel",
],
),
crate_features = [
"legacy",
"std",
"zdict_builder",
],
crate_root = "src/lib.rs",
edition = "2018",
rustc_flags = [
"--cap-lints=allow",
],
tags = [
"cargo-bazel",
"crate-name=zstd-sys",
"manual",
"noclippy",
"norustfmt",
],
target_compatible_with = select({
"@rules_rust//rust/platform:aarch64-apple-darwin": [],
"@rules_rust//rust/platform:aarch64-apple-ios": [],
"@rules_rust//rust/platform:aarch64-apple-ios-sim": [],
"@rules_rust//rust/platform:aarch64-linux-android": [],
"@rules_rust//rust/platform:aarch64-pc-windows-msvc": [],
"@rules_rust//rust/platform:aarch64-unknown-fuchsia": [],
"@rules_rust//rust/platform:aarch64-unknown-linux-gnu": [],
"@rules_rust//rust/platform:aarch64-unknown-nixos-gnu": [],
"@rules_rust//rust/platform:aarch64-unknown-nto-qnx710": [],
"@rules_rust//rust/platform:aarch64-unknown-uefi": [],
"@rules_rust//rust/platform:arm-unknown-linux-gnueabi": [],
"@rules_rust//rust/platform:armv7-linux-androideabi": [],
"@rules_rust//rust/platform:armv7-unknown-linux-gnueabi": [],
"@rules_rust//rust/platform:i686-apple-darwin": [],
"@rules_rust//rust/platform:i686-linux-android": [],
"@rules_rust//rust/platform:i686-pc-windows-msvc": [],
"@rules_rust//rust/platform:i686-unknown-freebsd": [],
"@rules_rust//rust/platform:i686-unknown-linux-gnu": [],
"@rules_rust//rust/platform:powerpc-unknown-linux-gnu": [],
"@rules_rust//rust/platform:riscv32imc-unknown-none-elf": [],
"@rules_rust//rust/platform:riscv64gc-unknown-none-elf": [],
"@rules_rust//rust/platform:s390x-unknown-linux-gnu": [],
"@rules_rust//rust/platform:thumbv7em-none-eabi": [],
"@rules_rust//rust/platform:thumbv8m.main-none-eabi": [],
"@rules_rust//rust/platform:wasm32-unknown-unknown": [],
"@rules_rust//rust/platform:wasm32-wasip1": [],
"@rules_rust//rust/platform:x86_64-apple-darwin": [],
"@rules_rust//rust/platform:x86_64-apple-ios": [],
"@rules_rust//rust/platform:x86_64-linux-android": [],
"@rules_rust//rust/platform:x86_64-pc-windows-msvc": [],
"@rules_rust//rust/platform:x86_64-unknown-freebsd": [],
"@rules_rust//rust/platform:x86_64-unknown-fuchsia": [],
"@rules_rust//rust/platform:x86_64-unknown-linux-gnu": [],
"@rules_rust//rust/platform:x86_64-unknown-nixos-gnu": [],
"@rules_rust//rust/platform:x86_64-unknown-none": [],
"@rules_rust//rust/platform:x86_64-unknown-uefi": [],
"//conditions:default": ["@platforms//:incompatible"],
}),
version = "2.0.15+zstd.1.5.7",
deps = [
"@vendor_ts__zstd-sys-2.0.15-zstd.1.5.7//:build_script_build",
],
)
cargo_build_script(
name = "_bs",
srcs = glob(
include = ["**/*.rs"],
allow_empty = True,
),
compile_data = glob(
include = ["**"],
allow_empty = True,
exclude = [
"**/* *",
"**/*.rs",
".tmp_git_root/**/*",
"BUILD",
"BUILD.bazel",
"WORKSPACE",
"WORKSPACE.bazel",
],
),
crate_features = [
"legacy",
"std",
"zdict_builder",
],
crate_name = "build_script_build",
crate_root = "build.rs",
data = glob(
include = ["**"],
allow_empty = True,
exclude = [
"**/* *",
".tmp_git_root/**/*",
"BUILD",
"BUILD.bazel",
"WORKSPACE",
"WORKSPACE.bazel",
],
),
edition = "2018",
links = "zstd",
pkg_name = "zstd-sys",
rustc_flags = [
"--cap-lints=allow",
],
tags = [
"cargo-bazel",
"crate-name=zstd-sys",
"manual",
"noclippy",
"norustfmt",
],
version = "2.0.15+zstd.1.5.7",
visibility = ["//visibility:private"],
deps = [
"@vendor_ts__cc-1.2.7//:cc",
"@vendor_ts__pkg-config-0.3.32//:pkg_config",
],
)
alias(
name = "build_script_build",
actual = ":_bs",
tags = ["manual"],
)

52
misc/bazel/3rdparty/tree_sitter_extractors_deps/defs.bzl generated vendored
View File

@@ -381,6 +381,7 @@ _NORMAL_DEPENDENCIES = {
"tracing": Label("@vendor_ts__tracing-0.1.41//:tracing"),
"tracing-subscriber": Label("@vendor_ts__tracing-subscriber-0.3.19//:tracing_subscriber"),
"tree-sitter": Label("@vendor_ts__tree-sitter-0.24.6//:tree_sitter"),
"zstd": Label("@vendor_ts__zstd-0.13.3//:zstd"),
},
},
}
@@ -1658,6 +1659,16 @@ def crate_repositories():
build_file = Label("//misc/bazel/3rdparty/tree_sitter_extractors_deps:BUILD.itoa-1.0.15.bazel"),
)
maybe(
http_archive,
name = "vendor_ts__jobserver-0.1.32",
sha256 = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0",
type = "tar.gz",
urls = ["https://static.crates.io/crates/jobserver/0.1.32/download"],
strip_prefix = "jobserver-0.1.32",
build_file = Label("//misc/bazel/3rdparty/tree_sitter_extractors_deps:BUILD.jobserver-0.1.32.bazel"),
)
maybe(
http_archive,
name = "vendor_ts__jod-thread-0.1.2",
@@ -2048,6 +2059,16 @@ def crate_repositories():
build_file = Label("//misc/bazel/3rdparty/tree_sitter_extractors_deps:BUILD.pin-project-lite-0.2.16.bazel"),
)
maybe(
http_archive,
name = "vendor_ts__pkg-config-0.3.32",
sha256 = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c",
type = "tar.gz",
urls = ["https://static.crates.io/crates/pkg-config/0.3.32/download"],
strip_prefix = "pkg-config-0.3.32",
build_file = Label("//misc/bazel/3rdparty/tree_sitter_extractors_deps:BUILD.pkg-config-0.3.32.bazel"),
)
maybe(
http_archive,
name = "vendor_ts__portable-atomic-1.11.0",
@@ -3647,6 +3668,36 @@ def crate_repositories():
build_file = Label("//misc/bazel/3rdparty/tree_sitter_extractors_deps:BUILD.zerocopy-derive-0.8.20.bazel"),
)
maybe(
http_archive,
name = "vendor_ts__zstd-0.13.3",
sha256 = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a",
type = "tar.gz",
urls = ["https://static.crates.io/crates/zstd/0.13.3/download"],
strip_prefix = "zstd-0.13.3",
build_file = Label("//misc/bazel/3rdparty/tree_sitter_extractors_deps:BUILD.zstd-0.13.3.bazel"),
)
maybe(
http_archive,
name = "vendor_ts__zstd-safe-7.2.4",
sha256 = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d",
type = "tar.gz",
urls = ["https://static.crates.io/crates/zstd-safe/7.2.4/download"],
strip_prefix = "zstd-safe-7.2.4",
build_file = Label("//misc/bazel/3rdparty/tree_sitter_extractors_deps:BUILD.zstd-safe-7.2.4.bazel"),
)
maybe(
http_archive,
name = "vendor_ts__zstd-sys-2.0.15-zstd.1.5.7",
sha256 = "eb81183ddd97d0c74cedf1d50d85c8d08c1b8b68ee863bdee9e706eedba1a237",
type = "tar.gz",
urls = ["https://static.crates.io/crates/zstd-sys/2.0.15+zstd.1.5.7/download"],
strip_prefix = "zstd-sys-2.0.15+zstd.1.5.7",
build_file = Label("//misc/bazel/3rdparty/tree_sitter_extractors_deps:BUILD.zstd-sys-2.0.15+zstd.1.5.7.bazel"),
)
return [
struct(repo = "vendor_ts__anyhow-1.0.97", is_dev_dep = False),
struct(repo = "vendor_ts__argfile-0.2.1", is_dev_dep = False),
@@ -3698,6 +3749,7 @@ def crate_repositories():
struct(repo = "vendor_ts__tree-sitter-ruby-0.23.1", is_dev_dep = False),
struct(repo = "vendor_ts__triomphe-0.1.14", is_dev_dep = False),
struct(repo = "vendor_ts__ungrammar-1.16.1", is_dev_dep = False),
struct(repo = "vendor_ts__zstd-0.13.3", is_dev_dep = False),
struct(repo = "vendor_ts__rand-0.9.0", is_dev_dep = True),
struct(repo = "vendor_ts__tree-sitter-json-0.24.8", is_dev_dep = True),
struct(repo = "vendor_ts__tree-sitter-ql-0.23.1", is_dev_dep = True),

6
ruby/codeql-extractor.yml
View File

@@ -27,7 +27,7 @@ options:
title: Controls compression for the TRAP files written by the extractor.
description: >
This option is only intended for use in debugging the extractor. Accepted
values are 'gzip' (the default, to write gzip-compressed TRAP) and 'none'
(to write uncompressed TRAP).
values are 'gzip' (the default, to write gzip-compressed TRAP) 'zstd' (to
write Zstandard-compressed TRAP) and 'none' (to write uncompressed TRAP).
type: string
pattern: "^(none|gzip)$"
pattern: "^(none|gzip|zstd)$"

1
ruby/ql/integration-tests/compression/methods.expected Normal file
View File

@@ -0,0 +1 @@
| source.rb:1:1:3:3 | f |

4
ruby/ql/integration-tests/compression/methods.ql Normal file
View File

@@ -0,0 +1,4 @@
import codeql.ruby.AST
from Method m
select m

3
ruby/ql/integration-tests/compression/source.rb Normal file
View File

@@ -0,0 +1,3 @@
def f
puts "hello"
end

26
ruby/ql/integration-tests/compression/test.py Normal file
View File

@@ -0,0 +1,26 @@
import pytest
@pytest.mark.parametrize(("compression", "suffix"), [
pytest.param("none", [], id="none"),
pytest.param("gzip", [".gz"], id="gzip"),
pytest.param("zstd", [".zst"], id="zstd"),
])
def test(codeql, ruby, compression, suffix, cwd):
codeql.database.create(
_env={
"CODEQL_EXTRACTOR_RUBY_OPTION_TRAP_COMPRESSION": compression,
}
)
trap_files = [*(cwd / "test-db" / "trap").rglob("*.trap*")]
assert trap_files, "No trap files found"
expected_suffixes = [".trap"] + suffix
def is_of_expected_format(file):
return file.name == "metadata.trap.gz" or \
file.suffixes[-len(expected_suffixes):] == expected_suffixes
files_with_wrong_format = [
f for f in trap_files if not is_of_expected_format(f)
]
assert not files_with_wrong_format, f"Found trap files with wrong format"

7
rust/codeql-extractor.yml
View File

@@ -23,10 +23,11 @@ options:
title: Controls compression for the TRAP files written by the extractor.
description: >
This option is only intended for use in debugging the extractor. Accepted
values are 'gzip' (to write gzip-compressed TRAP) and 'none'
(currently the default, to write uncompressed TRAP).
values are 'gzip' (to write gzip-compressed TRAP) 'zstd' (to write
Zstandard-compressed TRAP) and 'none' (the default, to write uncompressed
TRAP).
type: string
pattern: "^(none|gzip)$"
pattern: "^(none|gzip|zstd)$"
cargo_target_dir:
title: Directory to use for cargo output files.
description: >

4
rust/extractor/src/config.rs
View File

@@ -29,6 +29,7 @@ pub enum Compression {
#[default] // TODO make gzip default
None,
Gzip,
Zstd,
}
impl From<Compression> for trap::Compression {
@@ -36,6 +37,7 @@ impl From<Compression> for trap::Compression {
match val {
Compression::None => Self::None,
Compression::Gzip => Self::Gzip,
Compression::Zstd => Self::Zstd,
}
}
}
@@ -55,7 +57,7 @@ pub struct Config {
pub cargo_all_targets: bool,
pub logging_flamegraph: Option<PathBuf>,
pub logging_verbosity: Option<String>,
pub compression: Compression,
pub trap_compression: Compression,
pub inputs: Vec<PathBuf>,
pub qltest: bool,
pub qltest_cargo_check: bool,

14
rust/extractor/src/trap.rs
View File

@@ -1,4 +1,3 @@
use crate::config::Compression;
use crate::{config, generated};
use codeql_extractor::{extractor, file_paths, trap};
use ra_ap_ide_db::line_index::LineCol;
@@ -9,7 +8,7 @@ use std::path::{Path, PathBuf};
use tracing::debug;
pub use trap::Label as UntypedLabel;
pub use trap::Writer;
pub use trap::{Compression, Writer};
pub trait AsTrapKeyPart {
fn as_key_part(&self) -> String;
@@ -245,8 +244,7 @@ impl TrapFile {
pub fn commit(&self) -> std::io::Result<()> {
std::fs::create_dir_all(self.path.parent().unwrap())?;
self.writer
.write_to_file(&self.path, self.compression.into())
self.writer.write_to_file(&self.path, self.compression)
}
}
@@ -261,12 +259,16 @@ impl TrapFileProvider {
std::fs::create_dir_all(&trap_dir)?;
Ok(TrapFileProvider {
trap_dir,
compression: cfg.compression,
compression: cfg.trap_compression.into(),
})
}
pub fn create(&self, category: &str, key: impl AsRef<Path>) -> TrapFile {
let path = file_paths::path_for(&self.trap_dir.join(category), key.as_ref(), "trap");
let path = file_paths::path_for(
&self.trap_dir.join(category),
key.as_ref(),
self.compression.extension(),
);
debug!("creating trap file {}", path.display());
let mut writer = trap::Writer::new();
extractor::populate_empty_location(&mut writer);

29
rust/ql/integration-tests/hello-project/test_project.py
View File

@@ -10,7 +10,34 @@ def test_rust_project(codeql, rust, rust_project, check_source_archive, rust_che
codeql.database.create()
@pytest.mark.ql_test(None)
def test_do_not_print_env(codeql, rust, cargo, check_env_not_dumped, rust_check_diagnostics):
# parametrizing `rust_edition` allows us to skip the default parametrization over all editions
@pytest.mark.parametrize("rust_edition", [2024])
def test_do_not_print_env(codeql, rust, rust_edition, cargo, check_env_not_dumped, rust_check_diagnostics):
codeql.database.create(_env={
"CODEQL_EXTRACTOR_RUST_VERBOSE": "2",
})
@pytest.mark.ql_test("steps.ql", expected=".cargo.expected")
@pytest.mark.parametrize(("rust_edition", "compression", "suffix"), [
pytest.param(2024, "none", [], id="none"),
pytest.param(2024, "gzip", [".gz"], id="gzip"),
pytest.param(2024, "zstd", [".zst"], id="zstd"),
])
def test_compression(codeql, rust, rust_edition, compression, suffix, cargo, rust_check_diagnostics, cwd):
codeql.database.create(
_env={
"CODEQL_EXTRACTOR_RUST_OPTION_TRAP_COMPRESSION": compression,
}
)
trap_files = [*(cwd / "test-db" / "trap").rglob("*.trap*")]
assert trap_files, "No trap files found"
expected_suffixes = [".trap"] + suffix
def is_of_expected_format(file):
return file.name == "metadata.trap.gz" or \
file.suffixes[-len(expected_suffixes):] == expected_suffixes
files_with_wrong_format = [
f for f in trap_files if not is_of_expected_format(f)
]
assert not files_with_wrong_format, f"Found trap files with wrong format"

1
shared/tree-sitter-extractor/Cargo.toml
View File

@@ -19,6 +19,7 @@ serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
chrono = { version = "0.4.40", features = ["serde"] }
num_cpus = "1.16.0"
zstd = "0.13.3"
[dev-dependencies]
tree-sitter-ql = "0.23.1"

17
shared/tree-sitter-extractor/src/trap.rs
View File

@@ -96,10 +96,17 @@ impl Writer {
self.write_trap_entries(&mut trap_file)
}
Compression::Gzip => {
let trap_file = GzEncoder::new(trap_file, flate2::Compression::fast());
let trap_file = GzEncoder::new(trap_file, Compression::GZIP_LEVEL);
let mut trap_file = BufWriter::new(trap_file);
self.write_trap_entries(&mut trap_file)
}
Compression::Zstd => {
let trap_file = zstd::stream::Encoder::new(trap_file, Compression::ZSTD_LEVEL)?;
let mut trap_file = BufWriter::new(trap_file);
self.write_trap_entries(&mut trap_file)?;
trap_file.into_inner()?.finish()?;
Ok(())
}
}
}
@@ -107,7 +114,7 @@ impl Writer {
for trap_entry in &self.trap_output {
writeln!(file, "{}", trap_entry)?;
}
std::io::Result::Ok(())
Ok(())
}
}
@@ -280,9 +287,13 @@ fn limit_string(string: &str, max_size: usize) -> &str {
pub enum Compression {
None,
Gzip,
Zstd,
}
impl Compression {
pub const ZSTD_LEVEL: i32 = 2;
pub const GZIP_LEVEL: flate2::Compression = flate2::Compression::fast();
pub fn from_env(var_name: &str) -> Result<Compression, String> {
match std::env::var(var_name) {
Ok(method) => match Compression::from_string(&method) {
@@ -298,6 +309,7 @@ impl Compression {
match s.to_lowercase().as_ref() {
"none" => Some(Compression::None),
"gzip" => Some(Compression::Gzip),
"zstd" => Some(Compression::Zstd),
_ => None,
}
}
@@ -306,6 +318,7 @@ impl Compression {
match self {
Compression::None => "trap",
Compression::Gzip => "trap.gz",
Compression::Zstd => "trap.zst",
}
}
}