hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2025-12-16 08:43:11 +01:00
Code Issues Packages Projects Releases Wiki Activity

Bump tree-sitter to 0.23.0

Browse Source
This commit is contained in:
Tom Hvitved
2024-09-03 09:00:37 +02:00
parent 99400fe3d4
commit eb1b2a5594
21 changed files with 205 additions and 158 deletions
Download Patch File Download Diff File Expand all files Collapse all files

BIN
ql/Cargo.lock generated
View File

Binary file not shown.

2
ql/buramu/Cargo.toml
View File

@@ -1,7 +1,7 @@
[package] [package]
name = "buramu" name = "buramu"
version = "0.1.0" version = "0.1.0"
edition = "2018" edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

8
ql/buramu/tree-sitter-blame/Cargo.toml
View File

@@ -5,7 +5,7 @@ version = "0.0.1"
keywords = ["incremental", "parsing", "blame"] keywords = ["incremental", "parsing", "blame"]
categories = ["parsing", "text-editors"] categories = ["parsing", "text-editors"]
repository = "https://github.com/tree-sitter/tree-sitter-blame" repository = "https://github.com/tree-sitter/tree-sitter-blame"
edition = "2018" edition = "2021"
license = "MIT" license = "MIT"
build = "bindings/rust/build.rs" build = "bindings/rust/build.rs"
@@ -20,10 +20,8 @@ include = [
path = "bindings/rust/lib.rs" path = "bindings/rust/lib.rs"
[dependencies] [dependencies]
tree-sitter = ">= 0.22.6" tree-sitter-language = "0.1.0"
tree-sitter = ">= 0.23"
[build-dependencies] [build-dependencies]
cc = "1.0" cc = "1.0"
[patch.crates-io]
tree-sitter = {git = "https://github.com/redsun82/tree-sitter.git", rev = "1f5c1112ceaa8fc6aff61d1852690407670d2a96"}

23
ql/buramu/tree-sitter-blame/binding.gyp
View File

@@ -2,18 +2,29 @@
"targets": [ "targets": [
{ {
"target_name": "tree_sitter_blame_binding", "target_name": "tree_sitter_blame_binding",
"dependencies": [
"<!(node -p \"require('node-addon-api').targets\"):node_addon_api_except",
],
"include_dirs": [ "include_dirs": [
"<!(node -e \"require('nan')\")", "src",
"src"
], ],
"sources": [ "sources": [
"bindings/node/binding.cc", "bindings/node/binding.cc",
"src/parser.c", "src/parser.c",
# If your language uses an external scanner, add it here. # NOTE: if your language has an external scanner, add it here.
],
"conditions": [
["OS!='win'", {
"cflags_c": [
"-std=c11",
],
}, { # OS == "win"
"cflags_c": [
"/std:c11",
"/utf-8",
],
}],
], ],
"cflags_c": [
"-std=c99",
]
} }
] ]
} }

36
ql/buramu/tree-sitter-blame/bindings/node/binding.cc
View File

@@ -1,28 +1,20 @@
#include "tree_sitter/parser.h" #include <napi.h>
#include <node.h>
#include "nan.h"
using namespace v8; typedef struct TSLanguage TSLanguage;
extern "C" TSLanguage * tree_sitter_blame(); extern "C" TSLanguage *tree_sitter_blame();
namespace { // "tree-sitter", "language" hashed with BLAKE2
const napi_type_tag LANGUAGE_TYPE_TAG = {
0x8AF2E5212AD58ABF, 0xD5006CAD83ABBA16
};
NAN_METHOD(New) {} Napi::Object Init(Napi::Env env, Napi::Object exports) {
exports["name"] = Napi::String::New(env, "blame");
void Init(Local<Object> exports, Local<Object> module) { auto language = Napi::External<TSLanguage>::New(env, tree_sitter_blame());
Local<FunctionTemplate> tpl = Nan::New<FunctionTemplate>(New); language.TypeTag(&LANGUAGE_TYPE_TAG);
tpl->SetClassName(Nan::New("Language").ToLocalChecked()); exports["language"] = language;
tpl->InstanceTemplate()->SetInternalFieldCount(1); return exports;
Local<Function> constructor = Nan::GetFunction(tpl).ToLocalChecked();
Local<Object> instance = constructor->NewInstance(Nan::GetCurrentContext()).ToLocalChecked();
Nan::SetInternalFieldPointer(instance, 0, tree_sitter_blame());
Nan::Set(instance, Nan::New("name").ToLocalChecked(), Nan::New("blame").ToLocalChecked());
Nan::Set(module, Nan::New("exports").ToLocalChecked(), instance);
} }
NODE_MODULE(tree_sitter_blame_binding, Init) NODE_API_MODULE(tree_sitter_blame_binding, Init)
} // namespace

18
ql/buramu/tree-sitter-blame/bindings/node/index.js
View File

@@ -1,18 +1,6 @@
try { const root = require("path").join(__dirname, "..", "..");
module.exports = require("../../build/Release/tree_sitter_blame_binding");
} catch (error1) { module.exports = require("node-gyp-build")(root);
if (error1.code !== 'MODULE_NOT_FOUND') {
throw error1;
}
try {
module.exports = require("../../build/Debug/tree_sitter_blame_binding");
} catch (error2) {
if (error2.code !== 'MODULE_NOT_FOUND') {
throw error2;
}
throw error1
}
}
try { try {
module.exports.nodeTypeInfo = require("../../src/node-types.json"); module.exports.nodeTypeInfo = require("../../src/node-types.json");

3
ql/buramu/tree-sitter-blame/bindings/rust/build.rs
View File

@@ -7,6 +7,9 @@ fn main() {
.flag_if_supported("-Wno-unused-parameter") .flag_if_supported("-Wno-unused-parameter")
.flag_if_supported("-Wno-unused-but-set-variable") .flag_if_supported("-Wno-unused-but-set-variable")
.flag_if_supported("-Wno-trigraphs"); .flag_if_supported("-Wno-trigraphs");
#[cfg(target_env = "msvc")]
c_config.flag("-utf-8");
let parser_path = src_dir.join("parser.c"); let parser_path = src_dir.join("parser.c");
c_config.file(&parser_path); c_config.file(&parser_path);

41
ql/buramu/tree-sitter-blame/bindings/rust/lib.rs
View File

@@ -1,13 +1,18 @@
//! This crate provides blame language support for the [tree-sitter][] parsing library. //! This crate provides Blame language support for the [tree-sitter][] parsing library.
//! //!
//! Typically, you will use the [language][language func] function to add this language to a //! Typically, you will use the [language][language func] function to add this language to a
//! tree-sitter [Parser][], and then use the parser to parse some code: //! tree-sitter [Parser][], and then use the parser to parse some code:
//! //!
//! ``` //! ```ignore
//! let code = ""; //! let code = r#"
//! "#;
//! let mut parser = tree_sitter::Parser::new(); //! let mut parser = tree_sitter::Parser::new();
//! parser.set_language(tree_sitter_blame::language()).expect("Error loading blame grammar"); //! let language = tree_sitter_blame::LANGUAGE;
//! parser
//! .set_language(&language.into())
//! .expect("Error loading Blame parser"); // fails for some reason, so code block is ignored for now
//! let tree = parser.parse(code, None).unwrap(); //! let tree = parser.parse(code, None).unwrap();
//! assert!(!tree.root_node().has_error());
//! ``` //! ```
//! //!
//! [Language]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Language.html //! [Language]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Language.html
@@ -15,30 +20,26 @@
//! [Parser]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Parser.html //! [Parser]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Parser.html
//! [tree-sitter]: https://tree-sitter.github.io/ //! [tree-sitter]: https://tree-sitter.github.io/
use tree_sitter::Language; use tree_sitter_language::LanguageFn;
extern "C" { extern "C" {
fn tree_sitter_blame() -> Language; fn tree_sitter_blame() -> *const ();
} }
/// Get the tree-sitter [Language][] for this grammar. /// The tree-sitter [`LanguageFn`] for this grammar.
/// pub const LANGUAGE: LanguageFn = unsafe { LanguageFn::from_raw(tree_sitter_blame) };
/// [Language]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Language.html
pub fn language() -> Language {
unsafe { tree_sitter_blame() }
}
/// The content of the [`node-types.json`][] file for this grammar. /// The content of the [`node-types.json`][] file for this grammar.
/// ///
/// [`node-types.json`]: https://tree-sitter.github.io/tree-sitter/using-parsers#static-node-types /// [`node-types.json`]: https://tree-sitter.github.io/tree-sitter/using-parsers#static-node-types
pub const NODE_TYPES: &'static str = include_str!("../../src/node-types.json"); pub const NODE_TYPES: &str = include_str!("../../src/node-types.json");
// Uncomment these to include any queries that this grammar contains // NOTE: uncomment these to include any queries that this grammar contains:
// pub const HIGHLIGHTS_QUERY: &'static str = include_str!("../../queries/highlights.scm"); // pub const HIGHLIGHTS_QUERY: &str = include_str!("../../queries/highlights.scm");
// pub const INJECTIONS_QUERY: &'static str = include_str!("../../queries/injections.scm"); // pub const INJECTIONS_QUERY: &str = include_str!("../../queries/injections.scm");
// pub const LOCALS_QUERY: &'static str = include_str!("../../queries/locals.scm"); // pub const LOCALS_QUERY: &str = include_str!("../../queries/locals.scm");
// pub const TAGS_QUERY: &'static str = include_str!("../../queries/tags.scm"); // pub const TAGS_QUERY: &str = include_str!("../../queries/tags.scm");
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
@@ -46,7 +47,7 @@ mod tests {
fn test_can_load_grammar() { fn test_can_load_grammar() {
let mut parser = tree_sitter::Parser::new(); let mut parser = tree_sitter::Parser::new();
parser parser
.set_language(super::language()) .set_language(&super::LANGUAGE.into())
.expect("Error loading blame language"); .expect("Error loading Blame parser");
} }
} }

39
ql/buramu/tree-sitter-blame/package.json
View File

@@ -3,17 +3,46 @@
"version": "0.0.1", "version": "0.0.1",
"description": "blame grammar for tree-sitter", "description": "blame grammar for tree-sitter",
"main": "bindings/node", "main": "bindings/node",
"types": "bindings/node",
"keywords": [ "keywords": [
"parsing", "parsing",
"incremental" "incremental"
], ],
"files": [
"grammar.js",
"binding.gyp",
"prebuilds/**",
"bindings/node/*",
"queries/*",
"src/**",
"*.wasm"
],
"dependencies": { "dependencies": {
"nan": "^2.12.1" "node-addon-api": "^8.0.0",
"node-gyp-build": "^4.8.1"
},
"peerDependencies": {
"tree-sitter": "^0.23.0"
},
"peerDependenciesMeta": {
"tree_sitter": {
"optional": true
}
}, },
"devDependencies": { "devDependencies": {
"tree-sitter-cli": "^0.20.7" "tree-sitter-cli": "^0.23.0",
"prebuildify": "^6.0.1"
}, },
"scripts": { "scripts": {
"test": "tree-sitter test" "install": "node-gyp-build",
} "prestart": "tree-sitter build --wasm",
} "start": "tree-sitter playground",
"test": "node --test bindings/node/*_test.js"
},
"tree-sitter": [
{
"scope": "source.blame",
"injection-regex": "^blame$"
}
]
}

1
ql/buramu/tree-sitter-blame/src/grammar.json
View File

@@ -123,4 +123,3 @@
"inline": [], "inline": [],
"supertypes": [] "supertypes": []
} }

76
ql/buramu/tree-sitter-blame/src/parser.c
View File

@@ -1,7 +1,6 @@
#include <tree_sitter/parser.h> #include "tree_sitter/parser.h"
#if defined(__GNUC__) || defined(__clang__) #if defined(__GNUC__) || defined(__clang__)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wmissing-field-initializers" #pragma GCC diagnostic ignored "-Wmissing-field-initializers"
#endif #endif
@@ -16,7 +15,7 @@
#define MAX_ALIAS_SEQUENCE_LENGTH 4 #define MAX_ALIAS_SEQUENCE_LENGTH 4
#define PRODUCTION_ID_COUNT 14 #define PRODUCTION_ID_COUNT 14
enum { enum ts_symbol_identifiers {
anon_sym_today_COLON = 1, anon_sym_today_COLON = 1,
anon_sym_file_COLON = 2, anon_sym_file_COLON = 2,
anon_sym_LF = 3, anon_sym_LF = 3,
@@ -132,7 +131,7 @@ static const TSSymbolMetadata ts_symbol_metadata[] = {
}, },
}; };
enum { enum ts_field_identifiers {
field_blame_entry = 1, field_blame_entry = 1,
field_date = 2, field_date = 2,
field_file_entry = 3, field_file_entry = 3,
@@ -242,26 +241,21 @@ static bool ts_lex(TSLexer *lexer, TSStateId state) {
if (lookahead == 'f') ADVANCE(18); if (lookahead == 'f') ADVANCE(18);
if (lookahead == 'l') ADVANCE(10); if (lookahead == 'l') ADVANCE(10);
if (lookahead == 't') ADVANCE(23); if (lookahead == 't') ADVANCE(23);
if (lookahead == '\t' || if (('\t' <= lookahead && lookahead <= '\r') ||
lookahead == '\n' || lookahead == ' ') SKIP(0);
lookahead == '\r' ||
lookahead == ' ') SKIP(0)
if (('0' <= lookahead && lookahead <= '9')) ADVANCE(48); if (('0' <= lookahead && lookahead <= '9')) ADVANCE(48);
END_STATE(); END_STATE();
case 1: case 1:
if (lookahead == '\n') ADVANCE(40); if (lookahead == '\n') ADVANCE(40);
if (lookahead == '\t' || if (('\t' <= lookahead && lookahead <= '\r') ||
lookahead == '\r' || lookahead == ' ') SKIP(1);
lookahead == ' ') SKIP(1)
END_STATE(); END_STATE();
case 2: case 2:
if (lookahead == ' ') ADVANCE(39); if (lookahead == ' ') ADVANCE(39);
END_STATE(); END_STATE();
case 3: case 3:
if (lookahead == ' ') ADVANCE(43); if (lookahead == ' ') ADVANCE(43);
if (lookahead == '\t' || if (('\t' <= lookahead && lookahead <= '\r')) SKIP(3);
lookahead == '\n' ||
lookahead == '\r') SKIP(3)
if (('-' <= lookahead && lookahead <= '9') || if (('-' <= lookahead && lookahead <= '9') ||
('A' <= lookahead && lookahead <= 'Z') || ('A' <= lookahead && lookahead <= 'Z') ||
lookahead == '_' || lookahead == '_' ||
@@ -340,10 +334,8 @@ static bool ts_lex(TSLexer *lexer, TSStateId state) {
if (lookahead == 'y') ADVANCE(7); if (lookahead == 'y') ADVANCE(7);
END_STATE(); END_STATE();
case 28: case 28:
if (lookahead == '\t' || if (('\t' <= lookahead && lookahead <= '\r') ||
lookahead == '\n' || lookahead == ' ') SKIP(28);
lookahead == '\r' ||
lookahead == ' ') SKIP(28)
if (('0' <= lookahead && lookahead <= '9')) ADVANCE(35); if (('0' <= lookahead && lookahead <= '9')) ADVANCE(35);
END_STATE(); END_STATE();
case 29: case 29:
@@ -371,10 +363,8 @@ static bool ts_lex(TSLexer *lexer, TSStateId state) {
if (eof) ADVANCE(37); if (eof) ADVANCE(37);
if (lookahead == 'f') ADVANCE(18); if (lookahead == 'f') ADVANCE(18);
if (lookahead == 'l') ADVANCE(10); if (lookahead == 'l') ADVANCE(10);
if (lookahead == '\t' || if (('\t' <= lookahead && lookahead <= '\r') ||
lookahead == '\n' || lookahead == ' ') SKIP(36);
lookahead == '\r' ||
lookahead == ' ') SKIP(36)
if (('0' <= lookahead && lookahead <= '9')) ADVANCE(49); if (('0' <= lookahead && lookahead <= '9')) ADVANCE(49);
END_STATE(); END_STATE();
case 37: case 37:
@@ -623,25 +613,25 @@ static const TSParseActionEntry ts_parse_actions[] = {
[0] = {.entry = {.count = 0, .reusable = false}}, [0] = {.entry = {.count = 0, .reusable = false}},
[1] = {.entry = {.count = 1, .reusable = false}}, RECOVER(), [1] = {.entry = {.count = 1, .reusable = false}}, RECOVER(),
[3] = {.entry = {.count = 1, .reusable = true}}, SHIFT(15), [3] = {.entry = {.count = 1, .reusable = true}}, SHIFT(15),
[5] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_file_entry, 3, .production_id = 6), [5] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_file_entry, 3, 0, 6),
[7] = {.entry = {.count = 1, .reusable = true}}, SHIFT(19), [7] = {.entry = {.count = 1, .reusable = true}}, SHIFT(19),
[9] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_file_entry, 4, .production_id = 8), [9] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_file_entry, 4, 0, 8),
[11] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_blame_entry, 2, .production_id = 9), [11] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_blame_entry, 2, 0, 9),
[13] = {.entry = {.count = 1, .reusable = true}}, SHIFT(11), [13] = {.entry = {.count = 1, .reusable = true}}, SHIFT(11),
[15] = {.entry = {.count = 1, .reusable = true}}, REDUCE(aux_sym_file_entry_repeat1, 2, .production_id = 10), [15] = {.entry = {.count = 1, .reusable = true}}, REDUCE(aux_sym_file_entry_repeat1, 2, 0, 10),
[17] = {.entry = {.count = 2, .reusable = true}}, REDUCE(aux_sym_file_entry_repeat1, 2, .production_id = 10), SHIFT_REPEAT(19), [17] = {.entry = {.count = 2, .reusable = true}}, REDUCE(aux_sym_file_entry_repeat1, 2, 0, 10), SHIFT_REPEAT(19),
[20] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_blame_entry, 3, .production_id = 12), [20] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_blame_entry, 3, 0, 12),
[22] = {.entry = {.count = 1, .reusable = true}}, REDUCE(aux_sym_blame_entry_repeat1, 2, .production_id = 13), [22] = {.entry = {.count = 1, .reusable = true}}, REDUCE(aux_sym_blame_entry_repeat1, 2, 0, 13),
[24] = {.entry = {.count = 2, .reusable = true}}, REDUCE(aux_sym_blame_entry_repeat1, 2, .production_id = 13), SHIFT_REPEAT(11), [24] = {.entry = {.count = 2, .reusable = true}}, REDUCE(aux_sym_blame_entry_repeat1, 2, 0, 13), SHIFT_REPEAT(11),
[27] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_blame_info, 1, .production_id = 1), [27] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_blame_info, 1, 0, 1),
[29] = {.entry = {.count = 1, .reusable = true}}, SHIFT(17), [29] = {.entry = {.count = 1, .reusable = true}}, SHIFT(17),
[31] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_blame_info, 2, .production_id = 4), [31] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_blame_info, 2, 0, 4),
[33] = {.entry = {.count = 1, .reusable = true}}, REDUCE(aux_sym_blame_info_repeat1, 2, .production_id = 5), [33] = {.entry = {.count = 1, .reusable = true}}, REDUCE(aux_sym_blame_info_repeat1, 2, 0, 5),
[35] = {.entry = {.count = 2, .reusable = true}}, REDUCE(aux_sym_blame_info_repeat1, 2, .production_id = 5), SHIFT_REPEAT(17), [35] = {.entry = {.count = 2, .reusable = true}}, REDUCE(aux_sym_blame_info_repeat1, 2, 0, 5), SHIFT_REPEAT(17),
[38] = {.entry = {.count = 1, .reusable = true}}, REDUCE(aux_sym_blame_entry_repeat1, 1, .production_id = 11), [38] = {.entry = {.count = 1, .reusable = true}}, REDUCE(aux_sym_blame_entry_repeat1, 1, 0, 11),
[40] = {.entry = {.count = 1, .reusable = true}}, REDUCE(aux_sym_file_entry_repeat1, 1, .production_id = 7), [40] = {.entry = {.count = 1, .reusable = true}}, REDUCE(aux_sym_file_entry_repeat1, 1, 0, 7),
[42] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym__today, 2, .production_id = 2), [42] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym__today, 2, 0, 2),
[44] = {.entry = {.count = 1, .reusable = true}}, REDUCE(aux_sym_blame_info_repeat1, 1, .production_id = 3), [44] = {.entry = {.count = 1, .reusable = true}}, REDUCE(aux_sym_blame_info_repeat1, 1, 0, 3),
[46] = {.entry = {.count = 1, .reusable = true}}, SHIFT(13), [46] = {.entry = {.count = 1, .reusable = true}}, SHIFT(13),
[48] = {.entry = {.count = 1, .reusable = true}}, ACCEPT_INPUT(), [48] = {.entry = {.count = 1, .reusable = true}}, ACCEPT_INPUT(),
[50] = {.entry = {.count = 1, .reusable = true}}, SHIFT(18), [50] = {.entry = {.count = 1, .reusable = true}}, SHIFT(18),
@@ -652,11 +642,15 @@ static const TSParseActionEntry ts_parse_actions[] = {
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
#endif #endif
#ifdef _WIN32 #ifdef TREE_SITTER_HIDE_SYMBOLS
#define extern __declspec(dllexport) #define TS_PUBLIC
#elif defined(_WIN32)
#define TS_PUBLIC __declspec(dllexport)
#else
#define TS_PUBLIC __attribute__((visibility("default")))
#endif #endif
extern const TSLanguage *tree_sitter_blame(void) { TS_PUBLIC const TSLanguage *tree_sitter_blame(void) {
static const TSLanguage language = { static const TSLanguage language = {
.version = LANGUAGE_VERSION, .version = LANGUAGE_VERSION,
.symbol_count = SYMBOL_COUNT, .symbol_count = SYMBOL_COUNT,

68
ql/buramu/tree-sitter-blame/src/tree_sitter/parser.h
View File

@@ -13,9 +13,8 @@ extern "C" {
#define ts_builtin_sym_end 0 #define ts_builtin_sym_end 0
#define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024 #define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024
typedef uint16_t TSStateId;
#ifndef TREE_SITTER_API_H_ #ifndef TREE_SITTER_API_H_
typedef uint16_t TSStateId;
typedef uint16_t TSSymbol; typedef uint16_t TSSymbol;
typedef uint16_t TSFieldId; typedef uint16_t TSFieldId;
typedef struct TSLanguage TSLanguage; typedef struct TSLanguage TSLanguage;
@@ -48,6 +47,7 @@ struct TSLexer {
uint32_t (*get_column)(TSLexer *); uint32_t (*get_column)(TSLexer *);
bool (*is_at_included_range_start)(const TSLexer *); bool (*is_at_included_range_start)(const TSLexer *);
bool (*eof)(const TSLexer *); bool (*eof)(const TSLexer *);
void (*log)(const TSLexer *, const char *, ...);
}; };
typedef enum { typedef enum {
@@ -87,6 +87,11 @@ typedef union {
} entry; } entry;
} TSParseActionEntry; } TSParseActionEntry;
typedef struct {
int32_t start;
int32_t end;
} TSCharacterRange;
struct TSLanguage { struct TSLanguage {
uint32_t version; uint32_t version;
uint32_t symbol_count; uint32_t symbol_count;
@@ -126,13 +131,38 @@ struct TSLanguage {
const TSStateId *primary_state_ids; const TSStateId *primary_state_ids;
}; };
static inline bool set_contains(TSCharacterRange *ranges, uint32_t len, int32_t lookahead) {
uint32_t index = 0;
uint32_t size = len - index;
while (size > 1) {
uint32_t half_size = size / 2;
uint32_t mid_index = index + half_size;
TSCharacterRange *range = &ranges[mid_index];
if (lookahead >= range->start && lookahead <= range->end) {
return true;
} else if (lookahead > range->end) {
index = mid_index;
}
size -= half_size;
}
TSCharacterRange *range = &ranges[index];
return (lookahead >= range->start && lookahead <= range->end);
}
/* /*
* Lexer Macros * Lexer Macros
*/ */
#ifdef _MSC_VER
#define UNUSED __pragma(warning(suppress : 4101))
#else
#define UNUSED __attribute__((unused))
#endif
#define START_LEXER() \ #define START_LEXER() \
bool result = false; \ bool result = false; \
bool skip = false; \ bool skip = false; \
UNUSED \
bool eof = false; \ bool eof = false; \
int32_t lookahead; \ int32_t lookahead; \
goto start; \ goto start; \
@@ -148,6 +178,17 @@ struct TSLanguage {
goto next_state; \ goto next_state; \
} }
#define ADVANCE_MAP(...) \
{ \
static const uint16_t map[] = { __VA_ARGS__ }; \
for (uint32_t i = 0; i < sizeof(map) / sizeof(map[0]); i += 2) { \
if (map[i] == lookahead) { \
state = map[i + 1]; \
goto next_state; \
} \
} \
}
#define SKIP(state_value) \ #define SKIP(state_value) \
{ \ { \
skip = true; \ skip = true; \
@@ -166,7 +207,7 @@ struct TSLanguage {
* Parse Table Macros * Parse Table Macros
*/ */
#define SMALL_STATE(id) id - LARGE_STATE_COUNT #define SMALL_STATE(id) ((id) - LARGE_STATE_COUNT)
#define STATE(id) id #define STATE(id) id
@@ -176,7 +217,7 @@ struct TSLanguage {
{{ \ {{ \
.shift = { \ .shift = { \
.type = TSParseActionTypeShift, \ .type = TSParseActionTypeShift, \
.state = state_value \ .state = (state_value) \
} \ } \
}} }}
@@ -184,7 +225,7 @@ struct TSLanguage {
{{ \ {{ \
.shift = { \ .shift = { \
.type = TSParseActionTypeShift, \ .type = TSParseActionTypeShift, \
.state = state_value, \ .state = (state_value), \
.repetition = true \ .repetition = true \
} \ } \
}} }}
@@ -197,14 +238,15 @@ struct TSLanguage {
} \ } \
}} }}
#define REDUCE(symbol_val, child_count_val, ...) \ #define REDUCE(symbol_name, children, precedence, prod_id) \
{{ \ {{ \
.reduce = { \ .reduce = { \
.type = TSParseActionTypeReduce, \ .type = TSParseActionTypeReduce, \
.symbol = symbol_val, \ .symbol = symbol_name, \
.child_count = child_count_val, \ .child_count = children, \
__VA_ARGS__ \ .dynamic_precedence = precedence, \
}, \ .production_id = prod_id \
}, \
}} }}
#define RECOVER() \ #define RECOVER() \

11
ql/extractor/Cargo.toml
View File

@@ -7,17 +7,14 @@ edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies] [dependencies]
tree-sitter = ">= 0.22.6" tree-sitter = ">= 0.23.0"
tree-sitter-ql = { git = "https://github.com/tree-sitter/tree-sitter-ql.git", rev = "fa5c3821dd2161f5c8528a8cbdb258daa6dc4de6"} tree-sitter-ql = { git = "https://github.com/tree-sitter/tree-sitter-ql.git", rev = "c73c31c89cb0019ef56fe8bc1723e7c36e0be607"}
tree-sitter-ql-dbscheme = { git = "https://github.com/tree-sitter/tree-sitter-ql-dbscheme.git", rev = "5f770f57fa415607ff50e3d237d47c8f11440eb3"} tree-sitter-ql-dbscheme = { git = "https://github.com/tree-sitter/tree-sitter-ql-dbscheme.git", rev = "1980b4b6998a1138d326f863e6168f0f2c0c544d"}
tree-sitter-blame = {path = "../buramu/tree-sitter-blame"} tree-sitter-blame = {path = "../buramu/tree-sitter-blame"}
tree-sitter-json = {git = "https://github.com/tree-sitter/tree-sitter-json.git", rev = "94f5c527b2965465956c2000ed6134dd24daf2a7"} tree-sitter-json = {git = "https://github.com/tree-sitter/tree-sitter-json.git", rev = "8bfdb43f47ad805bb1ce093203cfcbaa8ed2c571"}
clap = { version = "4.2", features = ["derive"] } clap = { version = "4.2", features = ["derive"] }
tracing = "0.1" tracing = "0.1"
tracing-subscriber = { version = "0.3.18", features = ["env-filter"] } tracing-subscriber = { version = "0.3.18", features = ["env-filter"] }
rayon = "1.9.0" rayon = "1.9.0"
regex = "1.10.4" regex = "1.10.4"
codeql-extractor = { path = "../../shared/tree-sitter-extractor" } codeql-extractor = { path = "../../shared/tree-sitter-extractor" }
[patch.crates-io]
tree-sitter = {git = "https://github.com/redsun82/tree-sitter.git", rev = "1f5c1112ceaa8fc6aff61d1852690407670d2a96"}

8
ql/extractor/src/extractor.rs
View File

@@ -27,25 +27,25 @@ pub fn run(options: Options) -> std::io::Result<()> {
languages: vec![ languages: vec![
simple::LanguageSpec { simple::LanguageSpec {
prefix: "ql", prefix: "ql",
ts_language: tree_sitter_ql::language(), ts_language: tree_sitter_ql::LANGUAGE.into(),
node_types: tree_sitter_ql::NODE_TYPES, node_types: tree_sitter_ql::NODE_TYPES,
file_globs: vec!["*.ql".into(), "*.qll".into()], file_globs: vec!["*.ql".into(), "*.qll".into()],
}, },
simple::LanguageSpec { simple::LanguageSpec {
prefix: "dbscheme", prefix: "dbscheme",
ts_language: tree_sitter_ql_dbscheme::language(), ts_language: tree_sitter_ql_dbscheme::LANGUAGE.into(),
node_types: tree_sitter_ql_dbscheme::NODE_TYPES, node_types: tree_sitter_ql_dbscheme::NODE_TYPES,
file_globs: vec!["*.dbscheme".into()], file_globs: vec!["*.dbscheme".into()],
}, },
simple::LanguageSpec { simple::LanguageSpec {
prefix: "json", prefix: "json",
ts_language: tree_sitter_json::language(), ts_language: tree_sitter_json::LANGUAGE.into(),
node_types: tree_sitter_json::NODE_TYPES, node_types: tree_sitter_json::NODE_TYPES,
file_globs: vec!["*.json".into(), "*.jsonl".into(), "*.jsonc".into()], file_globs: vec!["*.json".into(), "*.jsonl".into(), "*.jsonc".into()],
}, },
simple::LanguageSpec { simple::LanguageSpec {
prefix: "blame", prefix: "blame",
ts_language: tree_sitter_blame::language(), ts_language: tree_sitter_blame::LANGUAGE.into(),
node_types: tree_sitter_blame::NODE_TYPES, node_types: tree_sitter_blame::NODE_TYPES,
file_globs: vec!["*.blame".into()], file_globs: vec!["*.blame".into()],
}, },

BIN
ruby/extractor/Cargo.lock generated
View File

Binary file not shown.

11
ruby/extractor/Cargo.toml
View File

@@ -7,20 +7,17 @@ authors = ["GitHub"]
edition = "2021" edition = "2021"
[dependencies] [dependencies]
tree-sitter = ">= 0.22.6" tree-sitter = ">= 0.23.0"
tree-sitter-embedded-template = { git = "https://github.com/tree-sitter/tree-sitter-embedded-template.git", rev = "38d5004a797298dc42c85e7706c5ceac46a3f29f" } tree-sitter-embedded-template = { git = "https://github.com/tree-sitter/tree-sitter-embedded-template.git", rev = "62b0a6e45900a7dff7c37da95fec20a09968ba52" }
tree-sitter-ruby = { git = "https://github.com/tree-sitter/tree-sitter-ruby.git", rev = "0ffe457fb6aabf064f173fd30ea356845cef2513" } tree-sitter-ruby = { git = "https://github.com/tree-sitter/tree-sitter-ruby.git", rev = "a66579f70d6f50ffd81a16fc3d3358e2ac173c88" }
clap = { version = "4.2", features = ["derive"] } clap = { version = "4.2", features = ["derive"] }
tracing = "0.1" tracing = "0.1"
tracing-subscriber = { version = "0.3.3", features = ["env-filter"] } tracing-subscriber = { version = "0.3.3", features = ["env-filter"] }
rayon = "1.5.0" rayon = "1.5.0"
regex = "1.7.1" regex = "1.7.1"
encoding = "0.2" encoding = "0.2"
lazy_static = "1.4.0" lazy_static = "1.4.0"
# Ideally, we'd like to pull this in via a relative path. # Ideally, we'd like to pull this in via a relative path.
# However, our bazel/rust tooling chokes on this, c.f. https://github.com/bazelbuild/rules_rust/issues/1525 # However, our bazel/rust tooling chokes on this, c.f. https://github.com/bazelbuild/rules_rust/issues/1525
# Therefore, we have a pretty bad hack in place instead, see README.md in the codeql-extractor-fake-crate directory. # Therefore, we have a pretty bad hack in place instead, see README.md in the codeql-extractor-fake-crate directory.
codeql-extractor = { path = "codeql-extractor-fake-crate" } codeql-extractor = { path = "codeql-extractor-fake-crate" }
[patch.crates-io]
tree-sitter = { git = "https://github.com/redsun82/tree-sitter.git", rev = "1f5c1112ceaa8fc6aff61d1852690407670d2a96" }

4
ruby/extractor/codeql-extractor-fake-crate/Cargo.toml
View File

@@ -7,7 +7,7 @@ authors = ["GitHub"]
[dependencies] [dependencies]
flate2 = "1.0" flate2 = "1.0"
globset = "0.4" globset = "0.4"
tree-sitter = ">= 0.22.6" tree-sitter = ">= 0.23.0"
tracing = "0.1" tracing = "0.1"
tracing-subscriber = { version = "0.3.3", features = ["env-filter"] } tracing-subscriber = { version = "0.3.3", features = ["env-filter"] }
rayon = "1.5.0" rayon = "1.5.0"
@@ -24,5 +24,3 @@ tree-sitter-ql = { git = "https://github.com/tree-sitter/tree-sitter-ql" }
tree-sitter-json = {git = "https://github.com/tree-sitter/tree-sitter-json" } tree-sitter-json = {git = "https://github.com/tree-sitter/tree-sitter-json" }
rand = "0.8.5" rand = "0.8.5"
[patch.crates-io]
tree-sitter = {git = "https://github.com/redsun82/tree-sitter.git", rev = "1f5c1112ceaa8fc6aff61d1852690407670d2a96"}

4
ruby/extractor/src/extractor.rs
View File

@@ -78,8 +78,8 @@ pub fn run(options: Options) -> std::io::Result<()> {
let file_list = fs::File::open(file_paths::path_from_string(&options.file_list))?; let file_list = fs::File::open(file_paths::path_from_string(&options.file_list))?;
let language = tree_sitter_ruby::language(); let language: Language = tree_sitter_ruby::LANGUAGE.into();
let erb = tree_sitter_embedded_template::language(); let erb: Language = tree_sitter_embedded_template::LANGUAGE.into();
// Look up tree-sitter kind ids now, to avoid string comparisons when scanning ERB files. // Look up tree-sitter kind ids now, to avoid string comparisons when scanning ERB files.
let erb_directive_id = erb.id_for_node_kind("directive", true); let erb_directive_id = erb.id_for_node_kind("directive", true);
let erb_output_directive_id = erb.id_for_node_kind("output_directive", true); let erb_output_directive_id = erb.id_for_node_kind("output_directive", true);

4
shared/tree-sitter-extractor/Cargo.toml
View File

@@ -7,7 +7,7 @@ authors = ["GitHub"]
[dependencies] [dependencies]
flate2 = "1.0" flate2 = "1.0"
globset = "0.4" globset = "0.4"
tree-sitter = ">= 0.22.6" tree-sitter = ">= 0.23.0"
tracing = "0.1" tracing = "0.1"
tracing-subscriber = { version = "0.3.3", features = ["env-filter"] } tracing-subscriber = { version = "0.3.3", features = ["env-filter"] }
rayon = "1.5.0" rayon = "1.5.0"
@@ -24,5 +24,3 @@ tree-sitter-ql = { git = "https://github.com/tree-sitter/tree-sitter-ql" }
tree-sitter-json = {git = "https://github.com/tree-sitter/tree-sitter-json" } tree-sitter-json = {git = "https://github.com/tree-sitter/tree-sitter-json" }
rand = "0.8.5" rand = "0.8.5"
[patch.crates-io]
tree-sitter = {git = "https://github.com/redsun82/tree-sitter.git", rev = "1f5c1112ceaa8fc6aff61d1852690407670d2a96"}

2
shared/tree-sitter-extractor/tests/integration_test.rs
View File

@@ -13,7 +13,7 @@ use common::{create_source_dir, expect_trap_file, SourceArchive};
fn simple_extractor() { fn simple_extractor() {
let language = simple::LanguageSpec { let language = simple::LanguageSpec {
prefix: "ql", prefix: "ql",
ts_language: tree_sitter_ql::language(), ts_language: tree_sitter_ql::LANGUAGE.into(),
node_types: tree_sitter_ql::NODE_TYPES, node_types: tree_sitter_ql::NODE_TYPES,
file_globs: vec!["*.qll".into()], file_globs: vec!["*.qll".into()],
}; };

4
shared/tree-sitter-extractor/tests/multiple_languages.rs
View File

@@ -12,13 +12,13 @@ use common::{create_source_dir, expect_trap_file, SourceArchive};
fn multiple_language_extractor() { fn multiple_language_extractor() {
let lang_ql = simple::LanguageSpec { let lang_ql = simple::LanguageSpec {
prefix: "ql", prefix: "ql",
ts_language: tree_sitter_ql::language(), ts_language: tree_sitter_ql::LANGUAGE.into(),
node_types: tree_sitter_ql::NODE_TYPES, node_types: tree_sitter_ql::NODE_TYPES,
file_globs: vec!["*.qll".into()], file_globs: vec!["*.qll".into()],
}; };
let lang_json = simple::LanguageSpec { let lang_json = simple::LanguageSpec {
prefix: "json", prefix: "json",
ts_language: tree_sitter_json::language(), ts_language: tree_sitter_json::LANGUAGE.into(),
node_types: tree_sitter_json::NODE_TYPES, node_types: tree_sitter_json::NODE_TYPES,
file_globs: vec!["*.json".into(), "*Jsonfile".into()], file_globs: vec!["*.json".into(), "*Jsonfile".into()],
}; };