hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2025-12-16 16:53:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

Tree-sitter: Allow for multiple file lists in simple extractor

Browse Source
This commit is contained in:
Tom Hvitved
2024-05-31 11:01:52 +02:00
parent 4c97b0c785
commit d6a3765597
4 changed files with 16 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
ql/extractor/src/extractor.rs
View File

@@ -53,7 +53,7 @@ pub fn run(options: Options) -> std::io::Result<()> {
trap_dir: options.output_dir, trap_dir: options.output_dir,
trap_compression: trap::Compression::from_env("CODEQL_QL_TRAP_COMPRESSION"), trap_compression: trap::Compression::from_env("CODEQL_QL_TRAP_COMPRESSION"),
source_archive_dir: options.source_archive_dir, source_archive_dir: options.source_archive_dir,
file_list: options.file_list, file_lists: vec![options.file_list],
}; };
extractor.run() extractor.run()

17
shared/tree-sitter-extractor/src/extractor/simple.rs
View File

@@ -20,7 +20,7 @@ pub struct Extractor {
pub languages: Vec<LanguageSpec>, pub languages: Vec<LanguageSpec>,
pub trap_dir: PathBuf, pub trap_dir: PathBuf,
pub source_archive_dir: PathBuf, pub source_archive_dir: PathBuf,
pub file_list: PathBuf, pub file_lists: Vec<PathBuf>,
// Typically constructed via `trap::Compression::from_env`. // Typically constructed via `trap::Compression::from_env`.
// This allow us to report the error using our diagnostics system // This allow us to report the error using our diagnostics system
// without exposing it to consumers. // without exposing it to consumers.
@@ -74,7 +74,14 @@ impl Extractor {
.build_global() .build_global()
.unwrap(); .unwrap();
let file_list = File::open(&self.file_list)?; let file_lists: Vec<File> = self
.file_lists
.iter()
.map(|file_list| {
File::open(file_list)
.unwrap_or_else(|_| panic!("Unable to open file list at {:?}", file_list))
})
.collect();
let mut schemas = vec![]; let mut schemas = vec![];
for lang in &self.languages { for lang in &self.languages {
@@ -103,8 +110,10 @@ impl Extractor {
) )
}; };
let lines: std::io::Result<Vec<String>> = let lines: std::io::Result<Vec<String>> = file_lists
std::io::BufReader::new(file_list).lines().collect(); .iter()
.flat_map(|file_list| std::io::BufReader::new(file_list).lines())
.collect();
let lines = lines?; let lines = lines?;
lines lines

2
shared/tree-sitter-extractor/tests/integration_test.rs
View File

@@ -30,7 +30,7 @@ fn simple_extractor() {
languages: vec![language], languages: vec![language],
trap_dir, trap_dir,
source_archive_dir, source_archive_dir,
file_list, file_lists: vec![file_list],
trap_compression: Ok(trap::Compression::Gzip), trap_compression: Ok(trap::Compression::Gzip),
}; };

2
shared/tree-sitter-extractor/tests/multiple_languages.rs
View File

@@ -39,7 +39,7 @@ fn multiple_language_extractor() {
languages: vec![lang_ql, lang_json], languages: vec![lang_ql, lang_json],
trap_dir, trap_dir,
source_archive_dir, source_archive_dir,
file_list, file_lists: vec![file_list],
trap_compression: Ok(trap::Compression::Gzip), trap_compression: Ok(trap::Compression::Gzip),
}; };