hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2025-12-16 16:53:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #19929 from github/tausbn/python-update-tree-sitter-dependency

Browse Source 
Python: Update `tree-sitter` dependency
This commit is contained in:
Taus
2025-09-17 13:40:13 +02:00
committed by GitHub
parent 4226fd29f5 f6732a927b
commit f5a06bef4a
17 changed files with 213 additions and 384 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
MODULE.bazel
View File

@@ -89,8 +89,8 @@ use_repo(
"vendor_py__cc-1.2.14", "vendor_py__cc-1.2.14",
"vendor_py__clap-4.5.30", "vendor_py__clap-4.5.30",
"vendor_py__regex-1.11.1", "vendor_py__regex-1.11.1",
"vendor_py__tree-sitter-0.20.4", "vendor_py__tree-sitter-0.24.7",
"vendor_py__tree-sitter-graph-0.7.0", "vendor_py__tree-sitter-graph-0.12.0",
) )
# deps for ruby+rust # deps for ruby+rust

12
misc/bazel/3rdparty/py_deps/BUILD.bazel generated vendored
View File

@@ -80,26 +80,26 @@ alias(
) )
alias( alias(
name = "tree-sitter-0.20.4", name = "tree-sitter-0.24.7",
actual = "@vendor_py__tree-sitter-0.20.4//:tree_sitter", actual = "@vendor_py__tree-sitter-0.24.7//:tree_sitter",
tags = ["manual"], tags = ["manual"],
) )
alias( alias(
name = "tree-sitter", name = "tree-sitter",
actual = "@vendor_py__tree-sitter-0.20.4//:tree_sitter", actual = "@vendor_py__tree-sitter-0.24.7//:tree_sitter",
tags = ["manual"], tags = ["manual"],
) )
alias( alias(
name = "tree-sitter-graph-0.7.0", name = "tree-sitter-graph-0.12.0",
actual = "@vendor_py__tree-sitter-graph-0.7.0//:tree_sitter_graph", actual = "@vendor_py__tree-sitter-graph-0.12.0//:tree_sitter_graph",
tags = ["manual"], tags = ["manual"],
) )
alias( alias(
name = "tree-sitter-graph", name = "tree-sitter-graph",
actual = "@vendor_py__tree-sitter-graph-0.7.0//:tree_sitter_graph", actual = "@vendor_py__tree-sitter-graph-0.12.0//:tree_sitter_graph",
tags = ["manual"], tags = ["manual"],
) )

99
misc/bazel/3rdparty/py_deps/BUILD.hashbrown-0.9.1.bazel generated vendored
View File

@@ -1,99 +0,0 @@
###############################################################################
# @generated
# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To
# regenerate this file, run the following:
#
# bazel run @@//misc/bazel/3rdparty:vendor_py_deps
###############################################################################
load("@rules_rust//cargo:defs.bzl", "cargo_toml_env_vars")
load("@rules_rust//rust:defs.bzl", "rust_library")
package(default_visibility = ["//visibility:public"])
cargo_toml_env_vars(
name = "cargo_toml_env_vars",
src = "Cargo.toml",
)
rust_library(
name = "hashbrown",
srcs = glob(
include = ["**/*.rs"],
allow_empty = True,
),
compile_data = glob(
include = ["**"],
allow_empty = True,
exclude = [
"**/* *",
".tmp_git_root/**/*",
"BUILD",
"BUILD.bazel",
"WORKSPACE",
"WORKSPACE.bazel",
],
),
crate_features = [
"ahash",
"inline-more",
],
crate_root = "src/lib.rs",
edition = "2018",
rustc_env_files = [
":cargo_toml_env_vars",
],
rustc_flags = [
"--cap-lints=allow",
],
tags = [
"cargo-bazel",
"crate-name=hashbrown",
"manual",
"noclippy",
"norustfmt",
],
target_compatible_with = select({
"@rules_rust//rust/platform:aarch64-apple-darwin": [],
"@rules_rust//rust/platform:aarch64-apple-ios": [],
"@rules_rust//rust/platform:aarch64-apple-ios-sim": [],
"@rules_rust//rust/platform:aarch64-linux-android": [],
"@rules_rust//rust/platform:aarch64-pc-windows-msvc": [],
"@rules_rust//rust/platform:aarch64-unknown-fuchsia": [],
"@rules_rust//rust/platform:aarch64-unknown-linux-gnu": [],
"@rules_rust//rust/platform:aarch64-unknown-nixos-gnu": [],
"@rules_rust//rust/platform:aarch64-unknown-nto-qnx710": [],
"@rules_rust//rust/platform:aarch64-unknown-uefi": [],
"@rules_rust//rust/platform:arm-unknown-linux-gnueabi": [],
"@rules_rust//rust/platform:armv7-linux-androideabi": [],
"@rules_rust//rust/platform:armv7-unknown-linux-gnueabi": [],
"@rules_rust//rust/platform:i686-apple-darwin": [],
"@rules_rust//rust/platform:i686-linux-android": [],
"@rules_rust//rust/platform:i686-pc-windows-msvc": [],
"@rules_rust//rust/platform:i686-unknown-freebsd": [],
"@rules_rust//rust/platform:i686-unknown-linux-gnu": [],
"@rules_rust//rust/platform:powerpc-unknown-linux-gnu": [],
"@rules_rust//rust/platform:riscv32imc-unknown-none-elf": [],
"@rules_rust//rust/platform:riscv64gc-unknown-none-elf": [],
"@rules_rust//rust/platform:s390x-unknown-linux-gnu": [],
"@rules_rust//rust/platform:thumbv7em-none-eabi": [],
"@rules_rust//rust/platform:thumbv8m.main-none-eabi": [],
"@rules_rust//rust/platform:wasm32-unknown-unknown": [],
"@rules_rust//rust/platform:wasm32-wasip1": [],
"@rules_rust//rust/platform:x86_64-apple-darwin": [],
"@rules_rust//rust/platform:x86_64-apple-ios": [],
"@rules_rust//rust/platform:x86_64-linux-android": [],
"@rules_rust//rust/platform:x86_64-pc-windows-msvc": [],
"@rules_rust//rust/platform:x86_64-unknown-freebsd": [],
"@rules_rust//rust/platform:x86_64-unknown-fuchsia": [],
"@rules_rust//rust/platform:x86_64-unknown-linux-gnu": [],
"@rules_rust//rust/platform:x86_64-unknown-nixos-gnu": [],
"@rules_rust//rust/platform:x86_64-unknown-none": [],
"@rules_rust//rust/platform:x86_64-unknown-uefi": [],
"//conditions:default": ["@platforms//:incompatible"],
}),
version = "0.9.1",
deps = [
"@vendor_py__ahash-0.4.8//:ahash",
],
)

8
misc/bazel/3rdparty/py_deps/BUILD.ahash-0.4.8.bazel → misc/bazel/3rdparty/py_deps/BUILD.streaming-iterator-0.1.9.bazel generated vendored
View File

@@ -17,7 +17,7 @@ cargo_toml_env_vars(
) )
rust_library( rust_library(
name = "ahash", name = "streaming_iterator",
srcs = glob( srcs = glob(
include = ["**/*.rs"], include = ["**/*.rs"],
allow_empty = True, allow_empty = True,
@@ -35,7 +35,7 @@ rust_library(
], ],
), ),
crate_root = "src/lib.rs", crate_root = "src/lib.rs",
edition = "2018", edition = "2021",
rustc_env_files = [ rustc_env_files = [
":cargo_toml_env_vars", ":cargo_toml_env_vars",
], ],
@@ -44,7 +44,7 @@ rust_library(
], ],
tags = [ tags = [
"cargo-bazel", "cargo-bazel",
"crate-name=ahash", "crate-name=streaming-iterator",
"manual", "manual",
"noclippy", "noclippy",
"norustfmt", "norustfmt",
@@ -88,5 +88,5 @@ rust_library(
"@rules_rust//rust/platform:x86_64-unknown-uefi": [], "@rules_rust//rust/platform:x86_64-unknown-uefi": [],
"//conditions:default": ["@platforms//:incompatible"], "//conditions:default": ["@platforms//:incompatible"],
}), }),
version = "0.4.8", version = "0.1.9",
) )

101
misc/bazel/3rdparty/py_deps/BUILD.string-interner-0.12.2.bazel generated vendored
View File

@@ -1,101 +0,0 @@
###############################################################################
# @generated
# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To
# regenerate this file, run the following:
#
# bazel run @@//misc/bazel/3rdparty:vendor_py_deps
###############################################################################
load("@rules_rust//cargo:defs.bzl", "cargo_toml_env_vars")
load("@rules_rust//rust:defs.bzl", "rust_library")
package(default_visibility = ["//visibility:public"])
cargo_toml_env_vars(
name = "cargo_toml_env_vars",
src = "Cargo.toml",
)
rust_library(
name = "string_interner",
srcs = glob(
include = ["**/*.rs"],
allow_empty = True,
),
compile_data = glob(
include = ["**"],
allow_empty = True,
exclude = [
"**/* *",
".tmp_git_root/**/*",
"BUILD",
"BUILD.bazel",
"WORKSPACE",
"WORKSPACE.bazel",
],
),
crate_features = [
"backends",
"inline-more",
"std",
],
crate_root = "src/lib.rs",
edition = "2018",
rustc_env_files = [
":cargo_toml_env_vars",
],
rustc_flags = [
"--cap-lints=allow",
],
tags = [
"cargo-bazel",
"crate-name=string-interner",
"manual",
"noclippy",
"norustfmt",
],
target_compatible_with = select({
"@rules_rust//rust/platform:aarch64-apple-darwin": [],
"@rules_rust//rust/platform:aarch64-apple-ios": [],
"@rules_rust//rust/platform:aarch64-apple-ios-sim": [],
"@rules_rust//rust/platform:aarch64-linux-android": [],
"@rules_rust//rust/platform:aarch64-pc-windows-msvc": [],
"@rules_rust//rust/platform:aarch64-unknown-fuchsia": [],
"@rules_rust//rust/platform:aarch64-unknown-linux-gnu": [],
"@rules_rust//rust/platform:aarch64-unknown-nixos-gnu": [],
"@rules_rust//rust/platform:aarch64-unknown-nto-qnx710": [],
"@rules_rust//rust/platform:aarch64-unknown-uefi": [],
"@rules_rust//rust/platform:arm-unknown-linux-gnueabi": [],
"@rules_rust//rust/platform:armv7-linux-androideabi": [],
"@rules_rust//rust/platform:armv7-unknown-linux-gnueabi": [],
"@rules_rust//rust/platform:i686-apple-darwin": [],
"@rules_rust//rust/platform:i686-linux-android": [],
"@rules_rust//rust/platform:i686-pc-windows-msvc": [],
"@rules_rust//rust/platform:i686-unknown-freebsd": [],
"@rules_rust//rust/platform:i686-unknown-linux-gnu": [],
"@rules_rust//rust/platform:powerpc-unknown-linux-gnu": [],
"@rules_rust//rust/platform:riscv32imc-unknown-none-elf": [],
"@rules_rust//rust/platform:riscv64gc-unknown-none-elf": [],
"@rules_rust//rust/platform:s390x-unknown-linux-gnu": [],
"@rules_rust//rust/platform:thumbv7em-none-eabi": [],
"@rules_rust//rust/platform:thumbv8m.main-none-eabi": [],
"@rules_rust//rust/platform:wasm32-unknown-unknown": [],
"@rules_rust//rust/platform:wasm32-wasip1": [],
"@rules_rust//rust/platform:x86_64-apple-darwin": [],
"@rules_rust//rust/platform:x86_64-apple-ios": [],
"@rules_rust//rust/platform:x86_64-linux-android": [],
"@rules_rust//rust/platform:x86_64-pc-windows-msvc": [],
"@rules_rust//rust/platform:x86_64-unknown-freebsd": [],
"@rules_rust//rust/platform:x86_64-unknown-fuchsia": [],
"@rules_rust//rust/platform:x86_64-unknown-linux-gnu": [],
"@rules_rust//rust/platform:x86_64-unknown-nixos-gnu": [],
"@rules_rust//rust/platform:x86_64-unknown-none": [],
"@rules_rust//rust/platform:x86_64-unknown-uefi": [],
"//conditions:default": ["@platforms//:incompatible"],
}),
version = "0.12.2",
deps = [
"@vendor_py__cfg-if-1.0.0//:cfg_if",
"@vendor_py__hashbrown-0.9.1//:hashbrown",
],
)

22
misc/bazel/3rdparty/py_deps/BUILD.tree-sitter-0.20.4.bazel → misc/bazel/3rdparty/py_deps/BUILD.tree-sitter-0.24.7.bazel generated vendored
View File

@@ -38,8 +38,12 @@ rust_library(
"WORKSPACE.bazel", "WORKSPACE.bazel",
], ],
), ),
crate_features = [
"default",
"std",
],
crate_root = "binding_rust/lib.rs", crate_root = "binding_rust/lib.rs",
edition = "2018", edition = "2021",
rustc_env_files = [ rustc_env_files = [
":cargo_toml_env_vars", ":cargo_toml_env_vars",
], ],
@@ -92,10 +96,13 @@ rust_library(
"@rules_rust//rust/platform:x86_64-unknown-uefi": [], "@rules_rust//rust/platform:x86_64-unknown-uefi": [],
"//conditions:default": ["@platforms//:incompatible"], "//conditions:default": ["@platforms//:incompatible"],
}), }),
version = "0.20.4", version = "0.24.7",
deps = [ deps = [
"@vendor_py__regex-1.11.1//:regex", "@vendor_py__regex-1.11.1//:regex",
"@vendor_py__tree-sitter-0.20.4//:build_script_build", "@vendor_py__regex-syntax-0.8.5//:regex_syntax",
"@vendor_py__streaming-iterator-0.1.9//:streaming_iterator",
"@vendor_py__tree-sitter-0.24.7//:build_script_build",
"@vendor_py__tree-sitter-language-0.1.5//:tree_sitter_language",
], ],
) )
@@ -118,6 +125,10 @@ cargo_build_script(
"WORKSPACE.bazel", "WORKSPACE.bazel",
], ],
), ),
crate_features = [
"default",
"std",
],
crate_name = "build_script_build", crate_name = "build_script_build",
crate_root = "binding_rust/build.rs", crate_root = "binding_rust/build.rs",
data = glob( data = glob(
@@ -132,7 +143,8 @@ cargo_build_script(
"WORKSPACE.bazel", "WORKSPACE.bazel",
], ],
), ),
edition = "2018", edition = "2021",
links = "tree-sitter",
pkg_name = "tree-sitter", pkg_name = "tree-sitter",
rustc_env_files = [ rustc_env_files = [
":cargo_toml_env_vars", ":cargo_toml_env_vars",
@@ -147,7 +159,7 @@ cargo_build_script(
"noclippy", "noclippy",
"norustfmt", "norustfmt",
], ],
version = "0.20.4", version = "0.24.7",
visibility = ["//visibility:private"], visibility = ["//visibility:private"],
deps = [ deps = [
"@vendor_py__cc-1.2.14//:cc", "@vendor_py__cc-1.2.14//:cc",

6
misc/bazel/3rdparty/py_deps/BUILD.tree-sitter-graph-0.7.0.bazel → misc/bazel/3rdparty/py_deps/BUILD.tree-sitter-graph-0.12.0.bazel generated vendored
View File

@@ -88,15 +88,15 @@ rust_library(
"@rules_rust//rust/platform:x86_64-unknown-uefi": [], "@rules_rust//rust/platform:x86_64-unknown-uefi": [],
"//conditions:default": ["@platforms//:incompatible"], "//conditions:default": ["@platforms//:incompatible"],
}), }),
version = "0.7.0", version = "0.12.0",
deps = [ deps = [
"@vendor_py__log-0.4.25//:log", "@vendor_py__log-0.4.25//:log",
"@vendor_py__regex-1.11.1//:regex", "@vendor_py__regex-1.11.1//:regex",
"@vendor_py__serde-1.0.217//:serde", "@vendor_py__serde-1.0.217//:serde",
"@vendor_py__serde_json-1.0.138//:serde_json", "@vendor_py__serde_json-1.0.138//:serde_json",
"@vendor_py__smallvec-1.14.0//:smallvec", "@vendor_py__smallvec-1.14.0//:smallvec",
"@vendor_py__string-interner-0.12.2//:string_interner", "@vendor_py__streaming-iterator-0.1.9//:streaming_iterator",
"@vendor_py__thiserror-1.0.69//:thiserror", "@vendor_py__thiserror-1.0.69//:thiserror",
"@vendor_py__tree-sitter-0.20.4//:tree_sitter", "@vendor_py__tree-sitter-0.24.7//:tree_sitter",
], ],
) )

10
misc/bazel/3rdparty/py_deps/BUILD.cfg-if-1.0.0.bazel → misc/bazel/3rdparty/py_deps/BUILD.tree-sitter-language-0.1.5.bazel generated vendored
View File

@@ -17,7 +17,7 @@ cargo_toml_env_vars(
) )
rust_library( rust_library(
name = "cfg_if", name = "tree_sitter_language",
srcs = glob( srcs = glob(
include = ["**/*.rs"], include = ["**/*.rs"],
allow_empty = True, allow_empty = True,
@@ -34,8 +34,8 @@ rust_library(
"WORKSPACE.bazel", "WORKSPACE.bazel",
], ],
), ),
crate_root = "src/lib.rs", crate_root = "language.rs",
edition = "2018", edition = "2021",
rustc_env_files = [ rustc_env_files = [
":cargo_toml_env_vars", ":cargo_toml_env_vars",
], ],
@@ -44,7 +44,7 @@ rust_library(
], ],
tags = [ tags = [
"cargo-bazel", "cargo-bazel",
"crate-name=cfg-if", "crate-name=tree-sitter-language",
"manual", "manual",
"noclippy", "noclippy",
"norustfmt", "norustfmt",
@@ -88,5 +88,5 @@ rust_library(
"@rules_rust//rust/platform:x86_64-unknown-uefi": [], "@rules_rust//rust/platform:x86_64-unknown-uefi": [],
"//conditions:default": ["@platforms//:incompatible"], "//conditions:default": ["@platforms//:incompatible"],
}), }),
version = "1.0.0", version = "0.1.5",
) )

80
misc/bazel/3rdparty/py_deps/defs.bzl generated vendored
View File

@@ -298,13 +298,13 @@ _NORMAL_DEPENDENCIES = {
"anyhow": Label("@vendor_py__anyhow-1.0.95//:anyhow"), "anyhow": Label("@vendor_py__anyhow-1.0.95//:anyhow"),
"clap": Label("@vendor_py__clap-4.5.30//:clap"), "clap": Label("@vendor_py__clap-4.5.30//:clap"),
"regex": Label("@vendor_py__regex-1.11.1//:regex"), "regex": Label("@vendor_py__regex-1.11.1//:regex"),
"tree-sitter": Label("@vendor_py__tree-sitter-0.20.4//:tree_sitter"), "tree-sitter": Label("@vendor_py__tree-sitter-0.24.7//:tree_sitter"),
"tree-sitter-graph": Label("@vendor_py__tree-sitter-graph-0.7.0//:tree_sitter_graph"), "tree-sitter-graph": Label("@vendor_py__tree-sitter-graph-0.12.0//:tree_sitter_graph"),
}, },
}, },
"python/extractor/tsg-python/tsp": { "python/extractor/tsg-python/tsp": {
_COMMON_CONDITION: { _COMMON_CONDITION: {
"tree-sitter": Label("@vendor_py__tree-sitter-0.20.4//:tree_sitter"), "tree-sitter": Label("@vendor_py__tree-sitter-0.24.7//:tree_sitter"),
}, },
}, },
} }
@@ -452,16 +452,6 @@ def crate_repositories():
Returns: Returns:
A list of repos visible to the module through the module extension. A list of repos visible to the module through the module extension.
""" """
maybe(
http_archive,
name = "vendor_py__ahash-0.4.8",
sha256 = "0453232ace82dee0dd0b4c87a59bd90f7b53b314f3e0f61fe2ee7c8a16482289",
type = "tar.gz",
urls = ["https://static.crates.io/crates/ahash/0.4.8/download"],
strip_prefix = "ahash-0.4.8",
build_file = Label("//misc/bazel/3rdparty/py_deps:BUILD.ahash-0.4.8.bazel"),
)
maybe( maybe(
http_archive, http_archive,
name = "vendor_py__aho-corasick-1.1.3", name = "vendor_py__aho-corasick-1.1.3",
@@ -542,16 +532,6 @@ def crate_repositories():
build_file = Label("//misc/bazel/3rdparty/py_deps:BUILD.cc-1.2.14.bazel"), build_file = Label("//misc/bazel/3rdparty/py_deps:BUILD.cc-1.2.14.bazel"),
) )
maybe(
http_archive,
name = "vendor_py__cfg-if-1.0.0",
sha256 = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd",
type = "tar.gz",
urls = ["https://static.crates.io/crates/cfg-if/1.0.0/download"],
strip_prefix = "cfg-if-1.0.0",
build_file = Label("//misc/bazel/3rdparty/py_deps:BUILD.cfg-if-1.0.0.bazel"),
)
maybe( maybe(
http_archive, http_archive,
name = "vendor_py__clap-4.5.30", name = "vendor_py__clap-4.5.30",
@@ -592,16 +572,6 @@ def crate_repositories():
build_file = Label("//misc/bazel/3rdparty/py_deps:BUILD.colorchoice-1.0.3.bazel"), build_file = Label("//misc/bazel/3rdparty/py_deps:BUILD.colorchoice-1.0.3.bazel"),
) )
maybe(
http_archive,
name = "vendor_py__hashbrown-0.9.1",
sha256 = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04",
type = "tar.gz",
urls = ["https://static.crates.io/crates/hashbrown/0.9.1/download"],
strip_prefix = "hashbrown-0.9.1",
build_file = Label("//misc/bazel/3rdparty/py_deps:BUILD.hashbrown-0.9.1.bazel"),
)
maybe( maybe(
http_archive, http_archive,
name = "vendor_py__is_terminal_polyfill-1.70.1", name = "vendor_py__is_terminal_polyfill-1.70.1",
@@ -764,12 +734,12 @@ def crate_repositories():
maybe( maybe(
http_archive, http_archive,
name = "vendor_py__string-interner-0.12.2", name = "vendor_py__streaming-iterator-0.1.9",
sha256 = "383196d1876517ee6f9f0864d1fc1070331b803335d3c6daaa04bbcccd823c08", sha256 = "2b2231b7c3057d5e4ad0156fb3dc807d900806020c5ffa3ee6ff2c8c76fb8520",
type = "tar.gz", type = "tar.gz",
urls = ["https://static.crates.io/crates/string-interner/0.12.2/download"], urls = ["https://static.crates.io/crates/streaming-iterator/0.1.9/download"],
strip_prefix = "string-interner-0.12.2", strip_prefix = "streaming-iterator-0.1.9",
build_file = Label("//misc/bazel/3rdparty/py_deps:BUILD.string-interner-0.12.2.bazel"), build_file = Label("//misc/bazel/3rdparty/py_deps:BUILD.streaming-iterator-0.1.9.bazel"),
) )
maybe( maybe(
@@ -814,22 +784,32 @@ def crate_repositories():
maybe( maybe(
http_archive, http_archive,
name = "vendor_py__tree-sitter-0.20.4", name = "vendor_py__tree-sitter-0.24.7",
sha256 = "4e34327f8eac545e3f037382471b2b19367725a242bba7bc45edb9efb49fe39a", sha256 = "a5387dffa7ffc7d2dae12b50c6f7aab8ff79d6210147c6613561fc3d474c6f75",
type = "tar.gz", type = "tar.gz",
urls = ["https://static.crates.io/crates/tree-sitter/0.20.4/download"], urls = ["https://static.crates.io/crates/tree-sitter/0.24.7/download"],
strip_prefix = "tree-sitter-0.20.4", strip_prefix = "tree-sitter-0.24.7",
build_file = Label("//misc/bazel/3rdparty/py_deps:BUILD.tree-sitter-0.20.4.bazel"), build_file = Label("//misc/bazel/3rdparty/py_deps:BUILD.tree-sitter-0.24.7.bazel"),
) )
maybe( maybe(
http_archive, http_archive,
name = "vendor_py__tree-sitter-graph-0.7.0", name = "vendor_py__tree-sitter-graph-0.12.0",
sha256 = "639d21e886f581d293de5f5081f09af003c54607ff3fa85efa159b243ba1f97a", sha256 = "63f86eb73c7d891c4b9b6fe4d4e63dd94c506e4788af7c2296afdcfbeea626cc",
type = "tar.gz", type = "tar.gz",
urls = ["https://static.crates.io/crates/tree-sitter-graph/0.7.0/download"], urls = ["https://static.crates.io/crates/tree-sitter-graph/0.12.0/download"],
strip_prefix = "tree-sitter-graph-0.7.0", strip_prefix = "tree-sitter-graph-0.12.0",
build_file = Label("//misc/bazel/3rdparty/py_deps:BUILD.tree-sitter-graph-0.7.0.bazel"), build_file = Label("//misc/bazel/3rdparty/py_deps:BUILD.tree-sitter-graph-0.12.0.bazel"),
)
maybe(
http_archive,
name = "vendor_py__tree-sitter-language-0.1.5",
sha256 = "c4013970217383f67b18aef68f6fb2e8d409bc5755227092d32efb0422ba24b8",
type = "tar.gz",
urls = ["https://static.crates.io/crates/tree-sitter-language/0.1.5/download"],
strip_prefix = "tree-sitter-language-0.1.5",
build_file = Label("//misc/bazel/3rdparty/py_deps:BUILD.tree-sitter-language-0.1.5.bazel"),
) )
maybe( maybe(
@@ -957,6 +937,6 @@ def crate_repositories():
struct(repo = "vendor_py__cc-1.2.14", is_dev_dep = False), struct(repo = "vendor_py__cc-1.2.14", is_dev_dep = False),
struct(repo = "vendor_py__clap-4.5.30", is_dev_dep = False), struct(repo = "vendor_py__clap-4.5.30", is_dev_dep = False),
struct(repo = "vendor_py__regex-1.11.1", is_dev_dep = False), struct(repo = "vendor_py__regex-1.11.1", is_dev_dep = False),
struct(repo = "vendor_py__tree-sitter-0.20.4", is_dev_dep = False), struct(repo = "vendor_py__tree-sitter-0.24.7", is_dev_dep = False),
struct(repo = "vendor_py__tree-sitter-graph-0.7.0", is_dev_dep = False), struct(repo = "vendor_py__tree-sitter-graph-0.12.0", is_dev_dep = False),
] ]

2
python/extractor/semmle/util.py
View File

@@ -10,7 +10,7 @@ from io import BytesIO
#Semantic version of extractor. #Semantic version of extractor.
#Update this if any changes are made #Update this if any changes are made
VERSION = "7.1.3" VERSION = "7.1.4"
PY_EXTENSIONS = ".py", ".pyw" PY_EXTENSIONS = ".py", ".pyw"

41
python/extractor/tests/parser/types_new.expected
View File

@@ -308,47 +308,6 @@ Module: [1, 0] - [23, 0]
] ]
ctx: Load ctx: Load
ctx: Load ctx: Load
TypeAlias: [20, 0] - [20, 41]
name:
Name: [20, 5] - [20, 9]
variable: Variable('Baz2', None)
ctx: Store
type_parameters: [
ParamSpec: [20, 10] - [20, 27]
name:
Name: [20, 12] - [20, 14]
variable: Variable('P2', None)
ctx: Store
default:
List: [20, 17] - [20, 27]
elts: [
Name: [20, 18] - [20, 21]
variable: Variable('int', None)
ctx: Load
Name: [20, 23] - [20, 26]
variable: Variable('str', None)
ctx: Load
]
ctx: Load
]
value:
Subscript: [20, 31] - [20, 41]
value:
Name: [20, 31] - [20, 35]
variable: Variable('Spam', None)
ctx: Load
index:
BinOp: [20, 36] - [20, 40]
left:
Name: [20, 36] - [20, 36]
variable: Variable('', None)
ctx: Load
op: Pow
right:
Name: [20, 38] - [20, 40]
variable: Variable('P2', None)
ctx: Load
ctx: Load
TypeAlias: [21, 0] - [21, 41] TypeAlias: [21, 0] - [21, 41]
name: name:
Name: [21, 5] - [21, 9] Name: [21, 5] - [21, 9]

2
python/extractor/tests/parser/types_new.py
View File

@@ -17,6 +17,6 @@ class Qux1[*Ts1 = *tuple[int, bool]]: ...
# TypeAliases # TypeAliases
type Foo2[T15, U1 = str] = Bar1[T15, U1] type Foo2[T15, U1 = str] = Bar1[T15, U1]
type Baz2[**P2 = [int, str]] = Spam[**P2] # type Baz2[**P2 = [int, str]] = Spam[**P2] # From the PEP, but this is not actually valid syntax!
type Qux2[*Ts2 = *tuple[str]] = Ham[*Ts2] type Qux2[*Ts2 = *tuple[str]] = Ham[*Ts2]
type Rab[U2, T15 = str] = Bar2[T15, U2] type Rab[U2, T15 = str] = Bar2[T15, U2]

52
python/extractor/tsg-python/Cargo.lock generated
View File

@@ -1,12 +1,6 @@
# This file is automatically @generated by Cargo. # This file is automatically @generated by Cargo.
# It is not intended for manual editing. # It is not intended for manual editing.
version = 3 version = 4
[[package]]
name = "ahash"
version = "0.4.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0453232ace82dee0dd0b4c87a59bd90f7b53b314f3e0f61fe2ee7c8a16482289"
[[package]] [[package]]
name = "aho-corasick" name = "aho-corasick"
@@ -82,12 +76,6 @@ dependencies = [
"shlex", "shlex",
] ]
[[package]]
name = "cfg-if"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]] [[package]]
name = "clap" name = "clap"
version = "4.5.30" version = "4.5.30"
@@ -121,15 +109,6 @@ version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990"
[[package]]
name = "hashbrown"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04"
dependencies = [
"ahash",
]
[[package]] [[package]]
name = "is_terminal_polyfill" name = "is_terminal_polyfill"
version = "1.70.1" version = "1.70.1"
@@ -258,14 +237,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7fcf8323ef1faaee30a44a340193b1ac6814fd9b7b4e88e9d4519a3e4abe1cfd" checksum = "7fcf8323ef1faaee30a44a340193b1ac6814fd9b7b4e88e9d4519a3e4abe1cfd"
[[package]] [[package]]
name = "string-interner" name = "streaming-iterator"
version = "0.12.2" version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "383196d1876517ee6f9f0864d1fc1070331b803335d3c6daaa04bbcccd823c08" checksum = "2b2231b7c3057d5e4ad0156fb3dc807d900806020c5ffa3ee6ff2c8c76fb8520"
dependencies = [
"cfg-if",
"hashbrown",
]
[[package]] [[package]]
name = "strsim" name = "strsim"
@@ -306,30 +281,39 @@ dependencies = [
[[package]] [[package]]
name = "tree-sitter" name = "tree-sitter"
version = "0.20.4" version = "0.24.7"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4e34327f8eac545e3f037382471b2b19367725a242bba7bc45edb9efb49fe39a" checksum = "a5387dffa7ffc7d2dae12b50c6f7aab8ff79d6210147c6613561fc3d474c6f75"
dependencies = [ dependencies = [
"cc", "cc",
"regex", "regex",
"regex-syntax",
"streaming-iterator",
"tree-sitter-language",
] ]
[[package]] [[package]]
name = "tree-sitter-graph" name = "tree-sitter-graph"
version = "0.7.0" version = "0.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "639d21e886f581d293de5f5081f09af003c54607ff3fa85efa159b243ba1f97a" checksum = "63f86eb73c7d891c4b9b6fe4d4e63dd94c506e4788af7c2296afdcfbeea626cc"
dependencies = [ dependencies = [
"log", "log",
"regex", "regex",
"serde", "serde",
"serde_json", "serde_json",
"smallvec", "smallvec",
"string-interner", "streaming-iterator",
"thiserror", "thiserror",
"tree-sitter", "tree-sitter",
] ]
[[package]]
name = "tree-sitter-language"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c4013970217383f67b18aef68f6fb2e8d409bc5755227092d32efb0422ba24b8"
[[package]] [[package]]
name = "tsg-python" name = "tsg-python"
version = "0.1.0" version = "0.1.0"

4
python/extractor/tsg-python/Cargo.toml
View File

@@ -10,7 +10,7 @@ edition = "2024"
[dependencies] [dependencies]
anyhow = "1.0" anyhow = "1.0"
regex = "1" regex = "1"
tree-sitter = "=0.20.4" tree-sitter = "=0.24.7"
tree-sitter-graph = "0.7.0" tree-sitter-graph = "0.12.0"
tsp = {path = "tsp"} tsp = {path = "tsp"}
clap = "4.5" clap = "4.5"

40
python/extractor/tsg-python/python.tsg
View File

@@ -6,16 +6,6 @@
(module) @mod (module) @mod
{ let @mod.node = (ast-node @mod "Module") } { let @mod.node = (ast-node @mod "Module") }
(_) @anynode
{
scan (node-type @anynode) {
"^(ERROR|MISSING)$" {
let @anynode.node = (ast-node @anynode "SyntaxErrorNode")
attr (@anynode.node) source = (source-text @anynode)
}
}
}
(parenthesized_expression) @nd (parenthesized_expression) @nd
{ let @nd.node = (ast-node @nd "Expr") } { let @nd.node = (ast-node @nd "Expr") }
@@ -416,13 +406,13 @@
attr (@if.node) _location_end = (location-end @expr) attr (@if.node) _location_end = (location-end @expr)
} }
(generator_expression . "(" . (comment)* . (expression) @start (for_in_clause) @child [(for_in_clause) (if_clause)] @end . (comment)* . ")" .) @genexpr (generator_expression . "(" . (comment)* . (expression) @start (for_in_clause) @child [(for_in_clause) (if_clause)] @end . (comment)* . ")" .) @_genexpr
{ {
attr (@child.node) _location_start = (location-start @start) attr (@child.node) _location_start = (location-start @start)
attr (@child.node) _location_end = (location-end @end) attr (@child.node) _location_end = (location-end @end)
} }
(generator_expression . "(" . (comment)* . (expression) @start (for_in_clause) @end . (comment)* . ")" .) @genexpr (generator_expression . "(" . (comment)* . (expression) @start (for_in_clause) @end . (comment)* . ")" .) @_genexpr
{ {
attr (@end.node) _location_start = (location-start @start) attr (@end.node) _location_start = (location-start @start)
attr (@end.node) _location_end = (location-end @end) attr (@end.node) _location_end = (location-end @end)
@@ -524,7 +514,7 @@
attr (@del.node -> @target.node) targets = (named-child-index @target) attr (@del.node -> @target.node) targets = (named-child-index @target)
} }
(delete_statement target: (_) @target) @del (delete_statement target: (_) @target) @_del
{ {
attr (@target.node) ctx = "del" attr (@target.node) ctx = "del"
} }
@@ -798,8 +788,8 @@
(dictionary_comprehension (dictionary_comprehension
body: (pair body: (pair
key: (_) @key key: (_) @_key
value: (_) @value value: (_) @_value
) )
) @genexpr ) @genexpr
{ {
@@ -1299,7 +1289,7 @@
; the index of the left-hand side of the current assignment. ; the index of the left-hand side of the current assignment.
; Base case, for the outermost assignment we set the outermost node to this node, and the index to zero. ; Base case, for the outermost assignment we set the outermost node to this node, and the index to zero.
(expression_statement (assignment !type) @assign) @expr (expression_statement (assignment !type) @assign) @_expr
{ {
let @assign.outermost_assignment = @assign.node let @assign.outermost_assignment = @assign.node
let @assign.target_index = 0 let @assign.target_index = 0
@@ -1358,7 +1348,7 @@
} }
(assignment (assignment
left: (_) @target left: (_) @_target
type: (_) type: (_)
right: (_) @value right: (_) @value
) @assign ) @assign
@@ -2330,7 +2320,7 @@
attr (@operand.node) ctx = "load" attr (@operand.node) ctx = "load"
} }
(unary_operator "~" @op) @unaryop (unary_operator "~" @_op) @unaryop
{ {
attr (@unaryop.node) op = "~" attr (@unaryop.node) op = "~"
} }
@@ -2614,7 +2604,7 @@
; Async status ; Async status
; NOTE: We only set the `is_async` field on the _first_ clause of the `with` statement, ; NOTE: We only set the `is_async` field on the _first_ clause of the `with` statement,
; as this is the behaviour of the old parser. ; as this is the behaviour of the old parser.
(with_statement "async" "with" @with_keyword (with_clause . (with_item) @with)) (with_statement "async" "with" @_with_keyword (with_clause . (with_item) @with))
{ {
attr (@with.node) is_async = #true attr (@with.node) is_async = #true
} }
@@ -2800,7 +2790,7 @@
(identifier) @obj (identifier) @obj
. .
(identifier) @attr (identifier) @attr
) @match_value_pattern ) @_match_value_pattern
{ {
let attribute = (ast-node @attr "Attribute") let attribute = (ast-node @attr "Attribute")
attr (@attr.node) _skip_to = attribute attr (@attr.node) _skip_to = attribute
@@ -2814,7 +2804,7 @@
(match_value_pattern (match_value_pattern
. .
(identifier) @id (identifier) @id
) @match_value_pattern ) @_match_value_pattern
{ {
attr (@id.node) ctx = "load" attr (@id.node) ctx = "load"
} }
@@ -3267,8 +3257,8 @@
(decorated_definition (decorated_definition
(decorator (expression) @exp1) @dec1 (decorator (expression) @exp1) @dec1
. (comment)* . . (comment)* .
(decorator (expression) @exp2) @dec2 (decorator (expression) @_exp2) @dec2
) @decorator ) @_decorator
{ {
attr (@dec1.node) func = @exp1.node attr (@dec1.node) func = @exp1.node
edge @dec1.node -> @dec2.node edge @dec1.node -> @dec2.node
@@ -3279,7 +3269,7 @@
(decorator (expression) @exp) @last (decorator (expression) @exp) @last
. (comment)* . . (comment)* .
definition: (function_definition) @funcdef definition: (function_definition) @funcdef
) @decorator ) @_decorator
{ {
attr (@last.node) func = @exp.node attr (@last.node) func = @exp.node
edge @last.node -> @funcdef.funcexpr edge @last.node -> @funcdef.funcexpr
@@ -3291,7 +3281,7 @@
(decorator (expression) @exp) @last (decorator (expression) @exp) @last
. (comment)* . . (comment)* .
definition: (class_definition) @class definition: (class_definition) @class
) @decorator ) @_decorator
{ {
attr (@last.node) func = @exp.node attr (@last.node) func = @exp.node
edge @last.node -> @class.class_expr edge @last.node -> @class.class_expr

112
python/extractor/tsg-python/src/main.rs
View File

@@ -480,6 +480,102 @@ pub mod extra_functions {
} }
} }
struct TreeIterator<'a> {
nodes_to_visit: Vec<tree_sitter::Node<'a>>,
}
impl<'a> TreeIterator<'a> {
fn new(root: tree_sitter::Node<'a>) -> Self {
Self {
nodes_to_visit: vec![root],
}
}
}
impl<'a> Iterator for TreeIterator<'a> {
type Item = tree_sitter::Node<'a>;
fn next(&mut self) -> Option<Self::Item> {
if let Some(node) = self.nodes_to_visit.pop() {
// Add all children to the queue for processing
let children: Vec<_> = (0..node.child_count())
.rev()
.filter_map(|i| node.child(i))
.collect();
self.nodes_to_visit.extend(children);
Some(node)
} else {
None
}
}
}
#[derive(Debug, Clone)]
struct SyntaxError {
start_pos: tree_sitter::Point,
end_pos: tree_sitter::Point,
source: String,
}
fn syntax_errors_from_tree<'a>(
root: tree_sitter::Node<'a>,
source: &'a str,
) -> impl Iterator<Item = SyntaxError> + 'a {
TreeIterator::new(root)
.filter(|&node| node.is_error() || node.is_missing())
.map(move |node| {
let start_pos = node.start_position();
let end_pos = node.end_position();
let text = &source.get(node.byte_range()).unwrap_or("");
SyntaxError {
start_pos,
end_pos,
source: text.to_string(),
}
})
}
fn add_syntax_error_nodes(graph: &mut tree_sitter_graph::graph::Graph, errors: &[SyntaxError]) {
for error in errors {
let error_node = graph.add_graph_node();
// Add _kind attribute
graph[error_node]
.attributes
.add(
tree_sitter_graph::Identifier::from("_kind"),
tree_sitter_graph::graph::Value::String("SyntaxErrorNode".to_string()),
)
.expect("Fresh node should not have duplicate attributes");
// Add _location attribute
let location = tree_sitter_graph::graph::Value::List(
vec![
error.start_pos.row,
error.start_pos.column,
error.end_pos.row,
error.end_pos.column,
]
.into_iter()
.map(|v| tree_sitter_graph::graph::Value::from(v as u32))
.collect(),
);
graph[error_node]
.attributes
.add(tree_sitter_graph::Identifier::from("_location"), location)
.expect("Fresh node should not have duplicate attributes");
// Add source attribute
graph[error_node]
.attributes
.add(
tree_sitter_graph::Identifier::from("source"),
tree_sitter_graph::graph::Value::String(error.source.clone()),
)
.expect("Fresh node should not have duplicate attributes");
}
}
fn main() -> Result<()> { fn main() -> Result<()> {
let matches = Command::new("tsg-python") let matches = Command::new("tsg-python")
.version(BUILD_VERSION) .version(BUILD_VERSION)
@@ -502,7 +598,7 @@ fn main() -> Result<()> {
let source_path = Path::new(matches.get_one::<String>("source").unwrap()); let source_path = Path::new(matches.get_one::<String>("source").unwrap());
let language = tsp::language(); let language = tsp::language();
let mut parser = Parser::new(); let mut parser = Parser::new();
parser.set_language(language)?; parser.set_language(&language)?;
// Statically include `python.tsg`: // Statically include `python.tsg`:
let tsg = if matches.contains_id("tsg") { let tsg = if matches.contains_id("tsg") {
std::fs::read(&tsg_path).with_context(|| format!("Error reading TSG file {}", tsg_path))? std::fs::read(&tsg_path).with_context(|| format!("Error reading TSG file {}", tsg_path))?
@@ -581,10 +677,18 @@ fn main() -> Result<()> {
); );
let globals = Variables::new(); let globals = Variables::new();
let mut config = ExecutionConfig::new(&mut functions, &globals).lazy(false); let config = ExecutionConfig::new(&functions, &globals).lazy(false);
let graph = file let mut graph = file
.execute(&tree, &source, &mut config, &NoCancellation) .execute(&tree, &source, &config, &NoCancellation)
.with_context(|| format!("Could not execute TSG file {}", tsg_path))?; .with_context(|| format!("Could not execute TSG file {}", tsg_path))?;
// Collect and add syntax error nodes to the graph
if tree.root_node().has_error() {
let syntax_errors: Vec<SyntaxError> =
syntax_errors_from_tree(tree.root_node(), &source).collect();
add_syntax_error_nodes(&mut graph, &syntax_errors);
}
print!("{}", graph.pretty_print()); print!("{}", graph.pretty_print());
Ok(()) Ok(())
} }

2
python/extractor/tsg-python/tsp/Cargo.toml
View File

@@ -26,7 +26,7 @@ path = "bindings/rust/lib.rs"
## When updating these dependencies, run `misc/bazel/3rdparty/update_cargo_deps.sh` ## When updating these dependencies, run `misc/bazel/3rdparty/update_cargo_deps.sh`
[dependencies] [dependencies]
tree-sitter = ">= 0.20, < 0.21" tree-sitter = "=0.24.7"
[build-dependencies] [build-dependencies]
cc = "1.2" cc = "1.2"