mirror of
https://github.com/github/codeql.git
synced 2026-07-31 15:33:00 +02:00
bbf52ce7a7
Previously the `simple` multi-language extractor carried an optional
desugarer, so every language (including plain tree-sitter ones such as
ql, dbscheme, json and blame) went through the same desugaring-aware
extraction path.
This commit splits it into two front-ends that share a private driver:
- `simple`: pure tree-sitter extraction with no desugaring. Comments
and other `extra` nodes are emitted inline as tokens. (The extractor
then extracts these as usual.)
- `desugaring`: parses source into a `ParsedTree` (a yeast AST plus
side-channel `extra` tokens) and rewrites the AST through a
`yeast::Desugarer` before extraction. The parser is a closure, so
both tree-sitter grammars (via `tree_sitter_parser`) and custom
parsers plug in the same way.
The shared multi-file plumbing (threading, glob matching, source-archive
copying, TRAP writing) lives in a new private `driver` module behind a
`LanguageExtractor` trait, so neither front-end duplicates it.
`extract` no longer takes an optional desugarer (it always walks the
parse tree directly); `extract_parsed` takes a required desugarer. ql
and ruby use the direct path; the unified Swift extractor uses the
desugaring path.
Also rename the new side-channel identifiers from "trivia" to "extra"
(ExtraToken, ParsedTree.extras, emit_extra, ...) to match tree-sitter's
own `is_extra()` terminology. The pre-existing `*_trivia_tokeninfo`
relation is left unchanged for a separate change.
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
39 lines
1.1 KiB
Rust
39 lines
1.1 KiB
Rust
use codeql_extractor::extractor::simple;
|
|
use codeql_extractor::trap;
|
|
|
|
mod common;
|
|
use common::{SourceArchive, create_source_dir, expect_trap_file};
|
|
|
|
/// A very simple happy-path test.
|
|
/// We run the extractor using the tree-sitter-ql grammar and a single source file,
|
|
/// and check that we get a reasonable-looking trap file in the expected location.
|
|
#[test]
|
|
fn simple_extractor() {
|
|
let language = simple::LanguageSpec {
|
|
prefix: "ql",
|
|
ts_language: tree_sitter_ql::LANGUAGE.into(),
|
|
node_types: tree_sitter_ql::NODE_TYPES,
|
|
file_globs: vec!["*.qll".into()],
|
|
};
|
|
|
|
let SourceArchive {
|
|
root_dir,
|
|
file_list,
|
|
source_archive_dir,
|
|
trap_dir,
|
|
} = create_source_dir(vec![("foo.qll", "predicate p(int a) { a = 1 }")]);
|
|
|
|
let extractor = simple::Extractor {
|
|
prefix: "ql".to_string(),
|
|
languages: vec![language],
|
|
trap_dir,
|
|
source_archive_dir,
|
|
file_lists: vec![file_list],
|
|
trap_compression: Ok(trap::Compression::Gzip),
|
|
};
|
|
|
|
extractor.run().unwrap();
|
|
|
|
expect_trap_file(&root_dir, "foo.qll");
|
|
}
|