Python: Copy Python extractor to codeql repo

python/extractor/tsg-python/tree-sitter-python/bindings/node/binding.cc

@@ -0,0 +1,28 @@
+#include "tree_sitter/parser.h"
+#include <node.h>
+#include "nan.h"
+
+using namespace v8;
+
+extern "C" TSLanguage * tree_sitter_python();
+
+namespace {
+
+NAN_METHOD(New) {}
+
+void Init(Local<Object> exports, Local<Object> module) {
+  Local<FunctionTemplate> tpl = Nan::New<FunctionTemplate>(New);
+  tpl->SetClassName(Nan::New("Language").ToLocalChecked());
+  tpl->InstanceTemplate()->SetInternalFieldCount(1);
+
+  Local<Function> constructor = Nan::GetFunction(tpl).ToLocalChecked();
+  Local<Object> instance = constructor->NewInstance(Nan::GetCurrentContext()).ToLocalChecked();
+  Nan::SetInternalFieldPointer(instance, 0, tree_sitter_python());
+
+  Nan::Set(instance, Nan::New("name").ToLocalChecked(), Nan::New("python").ToLocalChecked());
+  Nan::Set(module, Nan::New("exports").ToLocalChecked(), instance);
+}
+
+NODE_MODULE(tree_sitter_python_binding, Init)
+
+}  // namespace

python/extractor/tsg-python/tree-sitter-python/bindings/node/index.js

@@ -0,0 +1,19 @@
+try {
+  module.exports = require("../../build/Release/tree_sitter_python_binding");
+} catch (error1) {
+  if (error1.code !== 'MODULE_NOT_FOUND') {
+    throw error1;
+  }
+  try {
+    module.exports = require("../../build/Debug/tree_sitter_python_binding");
+  } catch (error2) {
+    if (error2.code !== 'MODULE_NOT_FOUND') {
+      throw error2;
+    }
+    throw error1
+  }
+}
+
+try {
+  module.exports.nodeTypeInfo = require("../../src/node-types.json");
+} catch (_) {}

python/extractor/tsg-python/tree-sitter-python/bindings/rust/README.md

@@ -0,0 +1,36 @@
+# tree-sitter-python
+
+This crate provides a Python grammar for the [tree-sitter][] parsing library.
+To use this crate, add it to the `[dependencies]` section of your `Cargo.toml`
+file.  (Note that you will probably also need to depend on the
+[`tree-sitter`][tree-sitter crate] crate to use the parsed result in any useful
+way.)
+
+``` toml
+[dependencies]
+tree-sitter = "0.17"
+tree-sitter-python = "0.17"
+```
+
+Typically, you will use the [language][language func] function to add this
+grammar to a tree-sitter [Parser][], and then use the parser to parse some code:
+
+``` rust
+let code = r#"
+    def double(x):
+        return x * 2
+"#;
+let mut parser = Parser::new();
+parser.set_language(tree_sitter_python::language()).expect("Error loading Python grammar");
+let parsed = parser.parse(code, None);
+```
+
+If you have any questions, please reach out to us in the [tree-sitter
+discussions] page.
+
+[Language]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Language.html
+[language func]: https://docs.rs/tree-sitter-python/*/tree_sitter_python/fn.language.html
+[Parser]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Parser.html
+[tree-sitter]: https://tree-sitter.github.io/
+[tree-sitter crate]: https://crates.io/crates/tree-sitter
+[tree-sitter discussions]: https://github.com/tree-sitter/tree-sitter/discussions

python/extractor/tsg-python/tree-sitter-python/bindings/rust/build.rs

@@ -0,0 +1,28 @@
+use std::path::Path;
+extern crate cc;
+
+fn main() {
+    let src_dir = Path::new("src");
+
+    let mut c_config = cc::Build::new();
+    c_config.include(&src_dir);
+    c_config
+        .flag_if_supported("-Wno-unused-parameter")
+        .flag_if_supported("-Wno-unused-but-set-variable")
+        .flag_if_supported("-Wno-trigraphs");
+    let parser_path = src_dir.join("parser.c");
+    c_config.file(&parser_path);
+    println!("cargo:rerun-if-changed={}", parser_path.to_str().unwrap());
+    c_config.compile("parser");
+
+    let mut cpp_config = cc::Build::new();
+    cpp_config.cpp(true);
+    cpp_config.include(&src_dir);
+    cpp_config
+        .flag_if_supported("-Wno-unused-parameter")
+        .flag_if_supported("-Wno-unused-but-set-variable");
+    let scanner_path = src_dir.join("scanner.cc");
+    cpp_config.file(&scanner_path);
+    println!("cargo:rerun-if-changed={}", scanner_path.to_str().unwrap());
+    cpp_config.compile("scanner");
+}

python/extractor/tsg-python/tree-sitter-python/bindings/rust/lib.rs

@@ -0,0 +1,68 @@
+// -*- coding: utf-8 -*-
+// ------------------------------------------------------------------------------------------------
+// Copyright © 2020, tree-sitter-python authors.
+// See the LICENSE file in this repo for license details.
+// ------------------------------------------------------------------------------------------------
+
+//! This crate provides a Python grammar for the [tree-sitter][] parsing library.
+//!
+//! Typically, you will use the [language][language func] function to add this grammar to a
+//! tree-sitter [Parser][], and then use the parser to parse some code:
+//!
+//! ```
+//! use tree_sitter::Parser;
+//!
+//! let code = r#"
+//!     def double(x):
+//!         return x * 2
+//! "#;
+//! let mut parser = Parser::new();
+//! parser.set_language(tree_sitter_python::language()).expect("Error loading Python grammar");
+//! let parsed = parser.parse(code, None);
+//! # let parsed = parsed.unwrap();
+//! # let root = parsed.root_node();
+//! # assert!(!root.has_error());
+//! ```
+//!
+//! [Language]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Language.html
+//! [language func]: fn.language.html
+//! [Parser]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Parser.html
+//! [tree-sitter]: https://tree-sitter.github.io/
+
+use tree_sitter::Language;
+
+extern "C" {
+    fn tree_sitter_python() -> Language;
+}
+
+/// Returns the tree-sitter [Language][] for this grammar.
+///
+/// [Language]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Language.html
+pub fn language() -> Language {
+    unsafe { tree_sitter_python() }
+}
+
+/// The source of the Python tree-sitter grammar description.
+pub const GRAMMAR: &'static str = include_str!("../../grammar.js");
+
+/// The syntax highlighting query for this language.
+pub const HIGHLIGHT_QUERY: &'static str = include_str!("../../queries/highlights.scm");
+
+/// The content of the [`node-types.json`][] file for this grammar.
+///
+/// [`node-types.json`]: https://tree-sitter.github.io/tree-sitter/using-parsers#static-node-types
+pub const NODE_TYPES: &'static str = include_str!("../../src/node-types.json");
+
+/// The symbol tagging query for this language.
+pub const TAGGING_QUERY: &'static str = include_str!("../../queries/tags.scm");
+
+#[cfg(test)]
+mod tests {
+    #[test]
+    fn can_load_grammar() {
+        let mut parser = tree_sitter::Parser::new();
+        parser
+            .set_language(super::language())
+            .expect("Error loading Python grammar");
+    }
+}