diff --git a/ql/Cargo.lock b/ql/Cargo.lock index ed04a4a8553..67a3f78fb41 100644 Binary files a/ql/Cargo.lock and b/ql/Cargo.lock differ diff --git a/ql/buramu/Cargo.toml b/ql/buramu/Cargo.toml index 741a9bc4337..03fdb49116a 100644 --- a/ql/buramu/Cargo.toml +++ b/ql/buramu/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "buramu" version = "0.1.0" -edition = "2018" +edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/ql/buramu/tree-sitter-blame/Cargo.toml b/ql/buramu/tree-sitter-blame/Cargo.toml index 9ff12014224..f48145488a3 100644 --- a/ql/buramu/tree-sitter-blame/Cargo.toml +++ b/ql/buramu/tree-sitter-blame/Cargo.toml @@ -5,7 +5,7 @@ version = "0.0.1" keywords = ["incremental", "parsing", "blame"] categories = ["parsing", "text-editors"] repository = "https://github.com/tree-sitter/tree-sitter-blame" -edition = "2018" +edition = "2021" license = "MIT" build = "bindings/rust/build.rs" @@ -20,10 +20,8 @@ include = [ path = "bindings/rust/lib.rs" [dependencies] -tree-sitter = ">= 0.22.6" +tree-sitter-language = "0.1.0" +tree-sitter = ">= 0.23" [build-dependencies] cc = "1.0" - -[patch.crates-io] -tree-sitter = {git = "https://github.com/redsun82/tree-sitter.git", rev = "1f5c1112ceaa8fc6aff61d1852690407670d2a96"} diff --git a/ql/buramu/tree-sitter-blame/binding.gyp b/ql/buramu/tree-sitter-blame/binding.gyp index d6cc395af07..27f3613f039 100644 --- a/ql/buramu/tree-sitter-blame/binding.gyp +++ b/ql/buramu/tree-sitter-blame/binding.gyp @@ -2,18 +2,29 @@ "targets": [ { "target_name": "tree_sitter_blame_binding", + "dependencies": [ + " -#include "nan.h" +#include -using namespace v8; +typedef struct TSLanguage TSLanguage; -extern "C" TSLanguage * tree_sitter_blame(); +extern "C" TSLanguage *tree_sitter_blame(); -namespace { +// "tree-sitter", "language" hashed with BLAKE2 +const napi_type_tag LANGUAGE_TYPE_TAG = { + 0x8AF2E5212AD58ABF, 0xD5006CAD83ABBA16 +}; -NAN_METHOD(New) {} - -void Init(Local exports, Local module) { - Local tpl = Nan::New(New); - tpl->SetClassName(Nan::New("Language").ToLocalChecked()); - tpl->InstanceTemplate()->SetInternalFieldCount(1); - - Local constructor = Nan::GetFunction(tpl).ToLocalChecked(); - Local instance = constructor->NewInstance(Nan::GetCurrentContext()).ToLocalChecked(); - Nan::SetInternalFieldPointer(instance, 0, tree_sitter_blame()); - - Nan::Set(instance, Nan::New("name").ToLocalChecked(), Nan::New("blame").ToLocalChecked()); - Nan::Set(module, Nan::New("exports").ToLocalChecked(), instance); +Napi::Object Init(Napi::Env env, Napi::Object exports) { + exports["name"] = Napi::String::New(env, "blame"); + auto language = Napi::External::New(env, tree_sitter_blame()); + language.TypeTag(&LANGUAGE_TYPE_TAG); + exports["language"] = language; + return exports; } -NODE_MODULE(tree_sitter_blame_binding, Init) - -} // namespace +NODE_API_MODULE(tree_sitter_blame_binding, Init) diff --git a/ql/buramu/tree-sitter-blame/bindings/node/index.js b/ql/buramu/tree-sitter-blame/bindings/node/index.js index 23af3bb321c..6657bcf42de 100644 --- a/ql/buramu/tree-sitter-blame/bindings/node/index.js +++ b/ql/buramu/tree-sitter-blame/bindings/node/index.js @@ -1,18 +1,6 @@ -try { - module.exports = require("../../build/Release/tree_sitter_blame_binding"); -} catch (error1) { - if (error1.code !== 'MODULE_NOT_FOUND') { - throw error1; - } - try { - module.exports = require("../../build/Debug/tree_sitter_blame_binding"); - } catch (error2) { - if (error2.code !== 'MODULE_NOT_FOUND') { - throw error2; - } - throw error1 - } -} +const root = require("path").join(__dirname, "..", ".."); + +module.exports = require("node-gyp-build")(root); try { module.exports.nodeTypeInfo = require("../../src/node-types.json"); diff --git a/ql/buramu/tree-sitter-blame/bindings/rust/build.rs b/ql/buramu/tree-sitter-blame/bindings/rust/build.rs index c6061f09953..4cc26f5e0f8 100644 --- a/ql/buramu/tree-sitter-blame/bindings/rust/build.rs +++ b/ql/buramu/tree-sitter-blame/bindings/rust/build.rs @@ -7,6 +7,9 @@ fn main() { .flag_if_supported("-Wno-unused-parameter") .flag_if_supported("-Wno-unused-but-set-variable") .flag_if_supported("-Wno-trigraphs"); + #[cfg(target_env = "msvc")] + c_config.flag("-utf-8"); + let parser_path = src_dir.join("parser.c"); c_config.file(&parser_path); diff --git a/ql/buramu/tree-sitter-blame/bindings/rust/lib.rs b/ql/buramu/tree-sitter-blame/bindings/rust/lib.rs index 268243b164f..6a0362de7d7 100644 --- a/ql/buramu/tree-sitter-blame/bindings/rust/lib.rs +++ b/ql/buramu/tree-sitter-blame/bindings/rust/lib.rs @@ -1,13 +1,18 @@ -//! This crate provides blame language support for the [tree-sitter][] parsing library. +//! This crate provides Blame language support for the [tree-sitter][] parsing library. //! //! Typically, you will use the [language][language func] function to add this language to a //! tree-sitter [Parser][], and then use the parser to parse some code: //! -//! ``` -//! let code = ""; +//! ```ignore +//! let code = r#" +//! "#; //! let mut parser = tree_sitter::Parser::new(); -//! parser.set_language(tree_sitter_blame::language()).expect("Error loading blame grammar"); +//! let language = tree_sitter_blame::LANGUAGE; +//! parser +//! .set_language(&language.into()) +//! .expect("Error loading Blame parser"); // fails for some reason, so code block is ignored for now //! let tree = parser.parse(code, None).unwrap(); +//! assert!(!tree.root_node().has_error()); //! ``` //! //! [Language]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Language.html @@ -15,30 +20,26 @@ //! [Parser]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Parser.html //! [tree-sitter]: https://tree-sitter.github.io/ -use tree_sitter::Language; +use tree_sitter_language::LanguageFn; extern "C" { - fn tree_sitter_blame() -> Language; + fn tree_sitter_blame() -> *const (); } -/// Get the tree-sitter [Language][] for this grammar. -/// -/// [Language]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Language.html -pub fn language() -> Language { - unsafe { tree_sitter_blame() } -} +/// The tree-sitter [`LanguageFn`] for this grammar. +pub const LANGUAGE: LanguageFn = unsafe { LanguageFn::from_raw(tree_sitter_blame) }; /// The content of the [`node-types.json`][] file for this grammar. /// /// [`node-types.json`]: https://tree-sitter.github.io/tree-sitter/using-parsers#static-node-types -pub const NODE_TYPES: &'static str = include_str!("../../src/node-types.json"); +pub const NODE_TYPES: &str = include_str!("../../src/node-types.json"); -// Uncomment these to include any queries that this grammar contains +// NOTE: uncomment these to include any queries that this grammar contains: -// pub const HIGHLIGHTS_QUERY: &'static str = include_str!("../../queries/highlights.scm"); -// pub const INJECTIONS_QUERY: &'static str = include_str!("../../queries/injections.scm"); -// pub const LOCALS_QUERY: &'static str = include_str!("../../queries/locals.scm"); -// pub const TAGS_QUERY: &'static str = include_str!("../../queries/tags.scm"); +// pub const HIGHLIGHTS_QUERY: &str = include_str!("../../queries/highlights.scm"); +// pub const INJECTIONS_QUERY: &str = include_str!("../../queries/injections.scm"); +// pub const LOCALS_QUERY: &str = include_str!("../../queries/locals.scm"); +// pub const TAGS_QUERY: &str = include_str!("../../queries/tags.scm"); #[cfg(test)] mod tests { @@ -46,7 +47,7 @@ mod tests { fn test_can_load_grammar() { let mut parser = tree_sitter::Parser::new(); parser - .set_language(super::language()) - .expect("Error loading blame language"); + .set_language(&super::LANGUAGE.into()) + .expect("Error loading Blame parser"); } } diff --git a/ql/buramu/tree-sitter-blame/package.json b/ql/buramu/tree-sitter-blame/package.json index f9517c74da5..ace46ffbf7c 100644 --- a/ql/buramu/tree-sitter-blame/package.json +++ b/ql/buramu/tree-sitter-blame/package.json @@ -3,17 +3,46 @@ "version": "0.0.1", "description": "blame grammar for tree-sitter", "main": "bindings/node", + "types": "bindings/node", "keywords": [ "parsing", "incremental" ], + "files": [ + "grammar.js", + "binding.gyp", + "prebuilds/**", + "bindings/node/*", + "queries/*", + "src/**", + "*.wasm" + ], "dependencies": { - "nan": "^2.12.1" + "node-addon-api": "^8.0.0", + "node-gyp-build": "^4.8.1" + }, + "peerDependencies": { + "tree-sitter": "^0.23.0" + }, + "peerDependenciesMeta": { + "tree_sitter": { + "optional": true + } }, "devDependencies": { - "tree-sitter-cli": "^0.20.7" + "tree-sitter-cli": "^0.23.0", + "prebuildify": "^6.0.1" }, "scripts": { - "test": "tree-sitter test" - } -} + "install": "node-gyp-build", + "prestart": "tree-sitter build --wasm", + "start": "tree-sitter playground", + "test": "node --test bindings/node/*_test.js" + }, + "tree-sitter": [ + { + "scope": "source.blame", + "injection-regex": "^blame$" + } + ] +} \ No newline at end of file diff --git a/ql/buramu/tree-sitter-blame/src/grammar.json b/ql/buramu/tree-sitter-blame/src/grammar.json index 9001554fd16..d1a31f01200 100644 --- a/ql/buramu/tree-sitter-blame/src/grammar.json +++ b/ql/buramu/tree-sitter-blame/src/grammar.json @@ -123,4 +123,3 @@ "inline": [], "supertypes": [] } - diff --git a/ql/buramu/tree-sitter-blame/src/parser.c b/ql/buramu/tree-sitter-blame/src/parser.c index 4803597f566..364bc16c8b3 100644 --- a/ql/buramu/tree-sitter-blame/src/parser.c +++ b/ql/buramu/tree-sitter-blame/src/parser.c @@ -1,7 +1,6 @@ -#include +#include "tree_sitter/parser.h" #if defined(__GNUC__) || defined(__clang__) -#pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wmissing-field-initializers" #endif @@ -16,7 +15,7 @@ #define MAX_ALIAS_SEQUENCE_LENGTH 4 #define PRODUCTION_ID_COUNT 14 -enum { +enum ts_symbol_identifiers { anon_sym_today_COLON = 1, anon_sym_file_COLON = 2, anon_sym_LF = 3, @@ -132,7 +131,7 @@ static const TSSymbolMetadata ts_symbol_metadata[] = { }, }; -enum { +enum ts_field_identifiers { field_blame_entry = 1, field_date = 2, field_file_entry = 3, @@ -242,26 +241,21 @@ static bool ts_lex(TSLexer *lexer, TSStateId state) { if (lookahead == 'f') ADVANCE(18); if (lookahead == 'l') ADVANCE(10); if (lookahead == 't') ADVANCE(23); - if (lookahead == '\t' || - lookahead == '\n' || - lookahead == '\r' || - lookahead == ' ') SKIP(0) + if (('\t' <= lookahead && lookahead <= '\r') || + lookahead == ' ') SKIP(0); if (('0' <= lookahead && lookahead <= '9')) ADVANCE(48); END_STATE(); case 1: if (lookahead == '\n') ADVANCE(40); - if (lookahead == '\t' || - lookahead == '\r' || - lookahead == ' ') SKIP(1) + if (('\t' <= lookahead && lookahead <= '\r') || + lookahead == ' ') SKIP(1); END_STATE(); case 2: if (lookahead == ' ') ADVANCE(39); END_STATE(); case 3: if (lookahead == ' ') ADVANCE(43); - if (lookahead == '\t' || - lookahead == '\n' || - lookahead == '\r') SKIP(3) + if (('\t' <= lookahead && lookahead <= '\r')) SKIP(3); if (('-' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || lookahead == '_' || @@ -340,10 +334,8 @@ static bool ts_lex(TSLexer *lexer, TSStateId state) { if (lookahead == 'y') ADVANCE(7); END_STATE(); case 28: - if (lookahead == '\t' || - lookahead == '\n' || - lookahead == '\r' || - lookahead == ' ') SKIP(28) + if (('\t' <= lookahead && lookahead <= '\r') || + lookahead == ' ') SKIP(28); if (('0' <= lookahead && lookahead <= '9')) ADVANCE(35); END_STATE(); case 29: @@ -371,10 +363,8 @@ static bool ts_lex(TSLexer *lexer, TSStateId state) { if (eof) ADVANCE(37); if (lookahead == 'f') ADVANCE(18); if (lookahead == 'l') ADVANCE(10); - if (lookahead == '\t' || - lookahead == '\n' || - lookahead == '\r' || - lookahead == ' ') SKIP(36) + if (('\t' <= lookahead && lookahead <= '\r') || + lookahead == ' ') SKIP(36); if (('0' <= lookahead && lookahead <= '9')) ADVANCE(49); END_STATE(); case 37: @@ -623,25 +613,25 @@ static const TSParseActionEntry ts_parse_actions[] = { [0] = {.entry = {.count = 0, .reusable = false}}, [1] = {.entry = {.count = 1, .reusable = false}}, RECOVER(), [3] = {.entry = {.count = 1, .reusable = true}}, SHIFT(15), - [5] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_file_entry, 3, .production_id = 6), + [5] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_file_entry, 3, 0, 6), [7] = {.entry = {.count = 1, .reusable = true}}, SHIFT(19), - [9] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_file_entry, 4, .production_id = 8), - [11] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_blame_entry, 2, .production_id = 9), + [9] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_file_entry, 4, 0, 8), + [11] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_blame_entry, 2, 0, 9), [13] = {.entry = {.count = 1, .reusable = true}}, SHIFT(11), - [15] = {.entry = {.count = 1, .reusable = true}}, REDUCE(aux_sym_file_entry_repeat1, 2, .production_id = 10), - [17] = {.entry = {.count = 2, .reusable = true}}, REDUCE(aux_sym_file_entry_repeat1, 2, .production_id = 10), SHIFT_REPEAT(19), - [20] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_blame_entry, 3, .production_id = 12), - [22] = {.entry = {.count = 1, .reusable = true}}, REDUCE(aux_sym_blame_entry_repeat1, 2, .production_id = 13), - [24] = {.entry = {.count = 2, .reusable = true}}, REDUCE(aux_sym_blame_entry_repeat1, 2, .production_id = 13), SHIFT_REPEAT(11), - [27] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_blame_info, 1, .production_id = 1), + [15] = {.entry = {.count = 1, .reusable = true}}, REDUCE(aux_sym_file_entry_repeat1, 2, 0, 10), + [17] = {.entry = {.count = 2, .reusable = true}}, REDUCE(aux_sym_file_entry_repeat1, 2, 0, 10), SHIFT_REPEAT(19), + [20] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_blame_entry, 3, 0, 12), + [22] = {.entry = {.count = 1, .reusable = true}}, REDUCE(aux_sym_blame_entry_repeat1, 2, 0, 13), + [24] = {.entry = {.count = 2, .reusable = true}}, REDUCE(aux_sym_blame_entry_repeat1, 2, 0, 13), SHIFT_REPEAT(11), + [27] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_blame_info, 1, 0, 1), [29] = {.entry = {.count = 1, .reusable = true}}, SHIFT(17), - [31] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_blame_info, 2, .production_id = 4), - [33] = {.entry = {.count = 1, .reusable = true}}, REDUCE(aux_sym_blame_info_repeat1, 2, .production_id = 5), - [35] = {.entry = {.count = 2, .reusable = true}}, REDUCE(aux_sym_blame_info_repeat1, 2, .production_id = 5), SHIFT_REPEAT(17), - [38] = {.entry = {.count = 1, .reusable = true}}, REDUCE(aux_sym_blame_entry_repeat1, 1, .production_id = 11), - [40] = {.entry = {.count = 1, .reusable = true}}, REDUCE(aux_sym_file_entry_repeat1, 1, .production_id = 7), - [42] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym__today, 2, .production_id = 2), - [44] = {.entry = {.count = 1, .reusable = true}}, REDUCE(aux_sym_blame_info_repeat1, 1, .production_id = 3), + [31] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_blame_info, 2, 0, 4), + [33] = {.entry = {.count = 1, .reusable = true}}, REDUCE(aux_sym_blame_info_repeat1, 2, 0, 5), + [35] = {.entry = {.count = 2, .reusable = true}}, REDUCE(aux_sym_blame_info_repeat1, 2, 0, 5), SHIFT_REPEAT(17), + [38] = {.entry = {.count = 1, .reusable = true}}, REDUCE(aux_sym_blame_entry_repeat1, 1, 0, 11), + [40] = {.entry = {.count = 1, .reusable = true}}, REDUCE(aux_sym_file_entry_repeat1, 1, 0, 7), + [42] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym__today, 2, 0, 2), + [44] = {.entry = {.count = 1, .reusable = true}}, REDUCE(aux_sym_blame_info_repeat1, 1, 0, 3), [46] = {.entry = {.count = 1, .reusable = true}}, SHIFT(13), [48] = {.entry = {.count = 1, .reusable = true}}, ACCEPT_INPUT(), [50] = {.entry = {.count = 1, .reusable = true}}, SHIFT(18), @@ -652,11 +642,15 @@ static const TSParseActionEntry ts_parse_actions[] = { #ifdef __cplusplus extern "C" { #endif -#ifdef _WIN32 -#define extern __declspec(dllexport) +#ifdef TREE_SITTER_HIDE_SYMBOLS +#define TS_PUBLIC +#elif defined(_WIN32) +#define TS_PUBLIC __declspec(dllexport) +#else +#define TS_PUBLIC __attribute__((visibility("default"))) #endif -extern const TSLanguage *tree_sitter_blame(void) { +TS_PUBLIC const TSLanguage *tree_sitter_blame(void) { static const TSLanguage language = { .version = LANGUAGE_VERSION, .symbol_count = SYMBOL_COUNT, diff --git a/ql/buramu/tree-sitter-blame/src/tree_sitter/parser.h b/ql/buramu/tree-sitter-blame/src/tree_sitter/parser.h index 2b14ac1046b..799f599bd4e 100644 --- a/ql/buramu/tree-sitter-blame/src/tree_sitter/parser.h +++ b/ql/buramu/tree-sitter-blame/src/tree_sitter/parser.h @@ -13,9 +13,8 @@ extern "C" { #define ts_builtin_sym_end 0 #define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024 -typedef uint16_t TSStateId; - #ifndef TREE_SITTER_API_H_ +typedef uint16_t TSStateId; typedef uint16_t TSSymbol; typedef uint16_t TSFieldId; typedef struct TSLanguage TSLanguage; @@ -48,6 +47,7 @@ struct TSLexer { uint32_t (*get_column)(TSLexer *); bool (*is_at_included_range_start)(const TSLexer *); bool (*eof)(const TSLexer *); + void (*log)(const TSLexer *, const char *, ...); }; typedef enum { @@ -87,6 +87,11 @@ typedef union { } entry; } TSParseActionEntry; +typedef struct { + int32_t start; + int32_t end; +} TSCharacterRange; + struct TSLanguage { uint32_t version; uint32_t symbol_count; @@ -126,13 +131,38 @@ struct TSLanguage { const TSStateId *primary_state_ids; }; +static inline bool set_contains(TSCharacterRange *ranges, uint32_t len, int32_t lookahead) { + uint32_t index = 0; + uint32_t size = len - index; + while (size > 1) { + uint32_t half_size = size / 2; + uint32_t mid_index = index + half_size; + TSCharacterRange *range = &ranges[mid_index]; + if (lookahead >= range->start && lookahead <= range->end) { + return true; + } else if (lookahead > range->end) { + index = mid_index; + } + size -= half_size; + } + TSCharacterRange *range = &ranges[index]; + return (lookahead >= range->start && lookahead <= range->end); +} + /* * Lexer Macros */ +#ifdef _MSC_VER +#define UNUSED __pragma(warning(suppress : 4101)) +#else +#define UNUSED __attribute__((unused)) +#endif + #define START_LEXER() \ bool result = false; \ bool skip = false; \ + UNUSED \ bool eof = false; \ int32_t lookahead; \ goto start; \ @@ -148,6 +178,17 @@ struct TSLanguage { goto next_state; \ } +#define ADVANCE_MAP(...) \ + { \ + static const uint16_t map[] = { __VA_ARGS__ }; \ + for (uint32_t i = 0; i < sizeof(map) / sizeof(map[0]); i += 2) { \ + if (map[i] == lookahead) { \ + state = map[i + 1]; \ + goto next_state; \ + } \ + } \ + } + #define SKIP(state_value) \ { \ skip = true; \ @@ -166,7 +207,7 @@ struct TSLanguage { * Parse Table Macros */ -#define SMALL_STATE(id) id - LARGE_STATE_COUNT +#define SMALL_STATE(id) ((id) - LARGE_STATE_COUNT) #define STATE(id) id @@ -176,7 +217,7 @@ struct TSLanguage { {{ \ .shift = { \ .type = TSParseActionTypeShift, \ - .state = state_value \ + .state = (state_value) \ } \ }} @@ -184,7 +225,7 @@ struct TSLanguage { {{ \ .shift = { \ .type = TSParseActionTypeShift, \ - .state = state_value, \ + .state = (state_value), \ .repetition = true \ } \ }} @@ -197,14 +238,15 @@ struct TSLanguage { } \ }} -#define REDUCE(symbol_val, child_count_val, ...) \ - {{ \ - .reduce = { \ - .type = TSParseActionTypeReduce, \ - .symbol = symbol_val, \ - .child_count = child_count_val, \ - __VA_ARGS__ \ - }, \ +#define REDUCE(symbol_name, children, precedence, prod_id) \ + {{ \ + .reduce = { \ + .type = TSParseActionTypeReduce, \ + .symbol = symbol_name, \ + .child_count = children, \ + .dynamic_precedence = precedence, \ + .production_id = prod_id \ + }, \ }} #define RECOVER() \ diff --git a/ql/extractor/Cargo.toml b/ql/extractor/Cargo.toml index 6f0e6e11f08..637a4bf5f3b 100644 --- a/ql/extractor/Cargo.toml +++ b/ql/extractor/Cargo.toml @@ -7,17 +7,14 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -tree-sitter = ">= 0.22.6" -tree-sitter-ql = { git = "https://github.com/tree-sitter/tree-sitter-ql.git", rev = "fa5c3821dd2161f5c8528a8cbdb258daa6dc4de6"} -tree-sitter-ql-dbscheme = { git = "https://github.com/tree-sitter/tree-sitter-ql-dbscheme.git", rev = "5f770f57fa415607ff50e3d237d47c8f11440eb3"} +tree-sitter = ">= 0.23.0" +tree-sitter-ql = { git = "https://github.com/tree-sitter/tree-sitter-ql.git", rev = "c73c31c89cb0019ef56fe8bc1723e7c36e0be607"} +tree-sitter-ql-dbscheme = { git = "https://github.com/tree-sitter/tree-sitter-ql-dbscheme.git", rev = "1980b4b6998a1138d326f863e6168f0f2c0c544d"} tree-sitter-blame = {path = "../buramu/tree-sitter-blame"} -tree-sitter-json = {git = "https://github.com/tree-sitter/tree-sitter-json.git", rev = "94f5c527b2965465956c2000ed6134dd24daf2a7"} +tree-sitter-json = {git = "https://github.com/tree-sitter/tree-sitter-json.git", rev = "8bfdb43f47ad805bb1ce093203cfcbaa8ed2c571"} clap = { version = "4.2", features = ["derive"] } tracing = "0.1" tracing-subscriber = { version = "0.3.18", features = ["env-filter"] } rayon = "1.9.0" regex = "1.10.4" codeql-extractor = { path = "../../shared/tree-sitter-extractor" } - -[patch.crates-io] -tree-sitter = {git = "https://github.com/redsun82/tree-sitter.git", rev = "1f5c1112ceaa8fc6aff61d1852690407670d2a96"} diff --git a/ql/extractor/src/extractor.rs b/ql/extractor/src/extractor.rs index b50cae32a01..8383d8424ee 100644 --- a/ql/extractor/src/extractor.rs +++ b/ql/extractor/src/extractor.rs @@ -27,25 +27,25 @@ pub fn run(options: Options) -> std::io::Result<()> { languages: vec![ simple::LanguageSpec { prefix: "ql", - ts_language: tree_sitter_ql::language(), + ts_language: tree_sitter_ql::LANGUAGE.into(), node_types: tree_sitter_ql::NODE_TYPES, file_globs: vec!["*.ql".into(), "*.qll".into()], }, simple::LanguageSpec { prefix: "dbscheme", - ts_language: tree_sitter_ql_dbscheme::language(), + ts_language: tree_sitter_ql_dbscheme::LANGUAGE.into(), node_types: tree_sitter_ql_dbscheme::NODE_TYPES, file_globs: vec!["*.dbscheme".into()], }, simple::LanguageSpec { prefix: "json", - ts_language: tree_sitter_json::language(), + ts_language: tree_sitter_json::LANGUAGE.into(), node_types: tree_sitter_json::NODE_TYPES, file_globs: vec!["*.json".into(), "*.jsonl".into(), "*.jsonc".into()], }, simple::LanguageSpec { prefix: "blame", - ts_language: tree_sitter_blame::language(), + ts_language: tree_sitter_blame::LANGUAGE.into(), node_types: tree_sitter_blame::NODE_TYPES, file_globs: vec!["*.blame".into()], }, diff --git a/ruby/extractor/Cargo.lock b/ruby/extractor/Cargo.lock index 95f32d9b610..d32d1bd6981 100644 Binary files a/ruby/extractor/Cargo.lock and b/ruby/extractor/Cargo.lock differ diff --git a/ruby/extractor/Cargo.toml b/ruby/extractor/Cargo.toml index 39310785ec3..89e0d3cf9db 100644 --- a/ruby/extractor/Cargo.toml +++ b/ruby/extractor/Cargo.toml @@ -7,20 +7,17 @@ authors = ["GitHub"] edition = "2021" [dependencies] -tree-sitter = ">= 0.22.6" -tree-sitter-embedded-template = { git = "https://github.com/tree-sitter/tree-sitter-embedded-template.git", rev = "38d5004a797298dc42c85e7706c5ceac46a3f29f" } -tree-sitter-ruby = { git = "https://github.com/tree-sitter/tree-sitter-ruby.git", rev = "0ffe457fb6aabf064f173fd30ea356845cef2513" } +tree-sitter = ">= 0.23.0" +tree-sitter-embedded-template = { git = "https://github.com/tree-sitter/tree-sitter-embedded-template.git", rev = "62b0a6e45900a7dff7c37da95fec20a09968ba52" } +tree-sitter-ruby = { git = "https://github.com/tree-sitter/tree-sitter-ruby.git", rev = "a66579f70d6f50ffd81a16fc3d3358e2ac173c88" } clap = { version = "4.2", features = ["derive"] } tracing = "0.1" tracing-subscriber = { version = "0.3.3", features = ["env-filter"] } rayon = "1.5.0" regex = "1.7.1" encoding = "0.2" -lazy_static = "1.4.0" +lazy_static = "1.4.0" # Ideally, we'd like to pull this in via a relative path. # However, our bazel/rust tooling chokes on this, c.f. https://github.com/bazelbuild/rules_rust/issues/1525 # Therefore, we have a pretty bad hack in place instead, see README.md in the codeql-extractor-fake-crate directory. codeql-extractor = { path = "codeql-extractor-fake-crate" } - -[patch.crates-io] -tree-sitter = { git = "https://github.com/redsun82/tree-sitter.git", rev = "1f5c1112ceaa8fc6aff61d1852690407670d2a96" } diff --git a/ruby/extractor/codeql-extractor-fake-crate/Cargo.toml b/ruby/extractor/codeql-extractor-fake-crate/Cargo.toml index d51d64a3349..d7d1e0f37ab 100644 --- a/ruby/extractor/codeql-extractor-fake-crate/Cargo.toml +++ b/ruby/extractor/codeql-extractor-fake-crate/Cargo.toml @@ -7,7 +7,7 @@ authors = ["GitHub"] [dependencies] flate2 = "1.0" globset = "0.4" -tree-sitter = ">= 0.22.6" +tree-sitter = ">= 0.23.0" tracing = "0.1" tracing-subscriber = { version = "0.3.3", features = ["env-filter"] } rayon = "1.5.0" @@ -24,5 +24,3 @@ tree-sitter-ql = { git = "https://github.com/tree-sitter/tree-sitter-ql" } tree-sitter-json = {git = "https://github.com/tree-sitter/tree-sitter-json" } rand = "0.8.5" -[patch.crates-io] -tree-sitter = {git = "https://github.com/redsun82/tree-sitter.git", rev = "1f5c1112ceaa8fc6aff61d1852690407670d2a96"} diff --git a/ruby/extractor/src/extractor.rs b/ruby/extractor/src/extractor.rs index b32e2d58194..f1c272f1792 100644 --- a/ruby/extractor/src/extractor.rs +++ b/ruby/extractor/src/extractor.rs @@ -78,8 +78,8 @@ pub fn run(options: Options) -> std::io::Result<()> { let file_list = fs::File::open(file_paths::path_from_string(&options.file_list))?; - let language = tree_sitter_ruby::language(); - let erb = tree_sitter_embedded_template::language(); + let language: Language = tree_sitter_ruby::LANGUAGE.into(); + let erb: Language = tree_sitter_embedded_template::LANGUAGE.into(); // Look up tree-sitter kind ids now, to avoid string comparisons when scanning ERB files. let erb_directive_id = erb.id_for_node_kind("directive", true); let erb_output_directive_id = erb.id_for_node_kind("output_directive", true); diff --git a/shared/tree-sitter-extractor/Cargo.toml b/shared/tree-sitter-extractor/Cargo.toml index d51d64a3349..d7d1e0f37ab 100644 --- a/shared/tree-sitter-extractor/Cargo.toml +++ b/shared/tree-sitter-extractor/Cargo.toml @@ -7,7 +7,7 @@ authors = ["GitHub"] [dependencies] flate2 = "1.0" globset = "0.4" -tree-sitter = ">= 0.22.6" +tree-sitter = ">= 0.23.0" tracing = "0.1" tracing-subscriber = { version = "0.3.3", features = ["env-filter"] } rayon = "1.5.0" @@ -24,5 +24,3 @@ tree-sitter-ql = { git = "https://github.com/tree-sitter/tree-sitter-ql" } tree-sitter-json = {git = "https://github.com/tree-sitter/tree-sitter-json" } rand = "0.8.5" -[patch.crates-io] -tree-sitter = {git = "https://github.com/redsun82/tree-sitter.git", rev = "1f5c1112ceaa8fc6aff61d1852690407670d2a96"} diff --git a/shared/tree-sitter-extractor/tests/integration_test.rs b/shared/tree-sitter-extractor/tests/integration_test.rs index 29058016764..cc453eeef74 100644 --- a/shared/tree-sitter-extractor/tests/integration_test.rs +++ b/shared/tree-sitter-extractor/tests/integration_test.rs @@ -13,7 +13,7 @@ use common::{create_source_dir, expect_trap_file, SourceArchive}; fn simple_extractor() { let language = simple::LanguageSpec { prefix: "ql", - ts_language: tree_sitter_ql::language(), + ts_language: tree_sitter_ql::LANGUAGE.into(), node_types: tree_sitter_ql::NODE_TYPES, file_globs: vec!["*.qll".into()], }; diff --git a/shared/tree-sitter-extractor/tests/multiple_languages.rs b/shared/tree-sitter-extractor/tests/multiple_languages.rs index 12a505f0170..f14100f6e0c 100644 --- a/shared/tree-sitter-extractor/tests/multiple_languages.rs +++ b/shared/tree-sitter-extractor/tests/multiple_languages.rs @@ -12,13 +12,13 @@ use common::{create_source_dir, expect_trap_file, SourceArchive}; fn multiple_language_extractor() { let lang_ql = simple::LanguageSpec { prefix: "ql", - ts_language: tree_sitter_ql::language(), + ts_language: tree_sitter_ql::LANGUAGE.into(), node_types: tree_sitter_ql::NODE_TYPES, file_globs: vec!["*.qll".into()], }; let lang_json = simple::LanguageSpec { prefix: "json", - ts_language: tree_sitter_json::language(), + ts_language: tree_sitter_json::LANGUAGE.into(), node_types: tree_sitter_json::NODE_TYPES, file_globs: vec!["*.json".into(), "*Jsonfile".into()], };