mirror of
https://github.com/github/codeql.git
synced 2026-07-31 15:33:00 +02:00
bbf52ce7a7
Previously the `simple` multi-language extractor carried an optional
desugarer, so every language (including plain tree-sitter ones such as
ql, dbscheme, json and blame) went through the same desugaring-aware
extraction path.
This commit splits it into two front-ends that share a private driver:
- `simple`: pure tree-sitter extraction with no desugaring. Comments
and other `extra` nodes are emitted inline as tokens. (The extractor
then extracts these as usual.)
- `desugaring`: parses source into a `ParsedTree` (a yeast AST plus
side-channel `extra` tokens) and rewrites the AST through a
`yeast::Desugarer` before extraction. The parser is a closure, so
both tree-sitter grammars (via `tree_sitter_parser`) and custom
parsers plug in the same way.
The shared multi-file plumbing (threading, glob matching, source-archive
copying, TRAP writing) lives in a new private `driver` module behind a
`LanguageExtractor` trait, so neither front-end duplicates it.
`extract` no longer takes an optional desugarer (it always walks the
parse tree directly); `extract_parsed` takes a required desugarer. ql
and ruby use the direct path; the unified Swift extractor uses the
desugaring path.
Also rename the new side-channel identifiers from "trivia" to "extra"
(ExtraToken, ParsedTree.extras, emit_extra, ...) to match tree-sitter's
own `is_extra()` terminology. The pre-existing `*_trivia_tokeninfo`
relation is left unchanged for a separate change.
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
51 lines
1.5 KiB
Rust
51 lines
1.5 KiB
Rust
use codeql_extractor::extractor::simple;
|
|
use codeql_extractor::trap;
|
|
|
|
mod common;
|
|
use common::{SourceArchive, create_source_dir, expect_trap_file};
|
|
|
|
/// Like the `simple_extractor` test but with multiple languages.
|
|
/// This is in a separate crate because the simple extractor API sets up a
|
|
/// global thread pool, and therefore can't be called twice in the same process.
|
|
#[test]
|
|
fn multiple_language_extractor() {
|
|
let lang_ql = simple::LanguageSpec {
|
|
prefix: "ql",
|
|
ts_language: tree_sitter_ql::LANGUAGE.into(),
|
|
node_types: tree_sitter_ql::NODE_TYPES,
|
|
file_globs: vec!["*.qll".into()],
|
|
};
|
|
let lang_json = simple::LanguageSpec {
|
|
prefix: "json",
|
|
ts_language: tree_sitter_json::LANGUAGE.into(),
|
|
node_types: tree_sitter_json::NODE_TYPES,
|
|
file_globs: vec!["*.json".into(), "*Jsonfile".into()],
|
|
};
|
|
|
|
let SourceArchive {
|
|
root_dir,
|
|
file_list,
|
|
source_archive_dir,
|
|
trap_dir,
|
|
} = create_source_dir(vec![
|
|
("foo.qll", "predicate p(int a) { a = 1 }"),
|
|
("bar.json", "{\"a\": 1}"),
|
|
("Jsonfile", "{\"b\": 2}"),
|
|
]);
|
|
|
|
let extractor = simple::Extractor {
|
|
prefix: "ql".to_string(),
|
|
languages: vec![lang_ql, lang_json],
|
|
trap_dir,
|
|
source_archive_dir,
|
|
file_lists: vec![file_list],
|
|
trap_compression: Ok(trap::Compression::Gzip),
|
|
};
|
|
|
|
extractor.run().unwrap();
|
|
|
|
expect_trap_file(&root_dir, "foo.qll");
|
|
expect_trap_file(&root_dir, "bar.json");
|
|
expect_trap_file(&root_dir, "Jsonfile");
|
|
}
|