hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2025-12-16 16:53:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #13029 from hmac/ruby-autobuilder-refactor

Browse Source 
Shared: Share autobuilder code between Ruby and QL
This commit is contained in:
Harry Maclean
2023-05-12 18:24:06 +07:00
committed by GitHub
parent c91d1cf721 9203efbdc4
commit 48f22681a5
4 changed files with 112 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

47
ql/extractor/src/autobuilder.rs
View File

@@ -1,48 +1,21 @@
use clap::Args;
use std::env;
use std::path::PathBuf;
use std::process::Command;
use clap::Args;
use codeql_extractor::autobuilder;
#[derive(Args)]
// The autobuilder takes no command-line options, but this may change in the future.
pub struct Options {}
pub fn run(_: Options) -> std::io::Result<()> {
let dist = env::var("CODEQL_DIST").expect("CODEQL_DIST not set");
let db = env::var("CODEQL_EXTRACTOR_QL_WIP_DATABASE")
let database = env::var("CODEQL_EXTRACTOR_QL_WIP_DATABASE")
.expect("CODEQL_EXTRACTOR_QL_WIP_DATABASE not set");
let codeql = if env::consts::OS == "windows" {
"codeql.exe"
} else {
"codeql"
};
let codeql: PathBuf = [&dist, codeql].iter().collect();
let mut cmd = Command::new(codeql);
cmd.arg("database")
.arg("index-files")
.arg("--include-extension=.ql")
.arg("--include-extension=.qll")
.arg("--include-extension=.dbscheme")
.arg("--include-extension=.json")
.arg("--include-extension=.jsonc")
.arg("--include-extension=.jsonl")
.arg("--include=**/qlpack.yml")
.arg("--include=deprecated.blame")
.arg("--size-limit=10m")
.arg("--language=ql")
.arg("--working-dir=.")
.arg(db);
for line in env::var("LGTM_INDEX_FILTERS")
.unwrap_or_default()
.split('\n')
{
if let Some(stripped) = line.strip_prefix("include:") {
cmd.arg("--also-match=".to_owned() + stripped);
} else if let Some(stripped) = line.strip_prefix("exclude:") {
cmd.arg("--exclude=".to_owned() + stripped);
}
}
let exit = &cmd.spawn()?.wait()?;
std::process::exit(exit.code().unwrap_or(1))
autobuilder::Autobuilder::new("ql", PathBuf::from(database))
.include_extensions(&[".ql", ".qll", ".dbscheme", ".json", ".jsonc", ".jsonl"])
.include_globs(&["**/qlpack.yml", "deprecated.blame"])
.size_limit("10m")
.run()
}

45
ruby/extractor/src/autobuilder.rs
View File

@@ -1,45 +1,22 @@
use clap::Args;
use std::env;
use std::path::PathBuf;
use std::process::Command;
use clap::Args;
use codeql_extractor::autobuilder;
#[derive(Args)]
// The autobuilder takes no command-line options, but this may change in the future.
pub struct Options {}
pub fn run(_: Options) -> std::io::Result<()> {
let dist = env::var("CODEQL_DIST").expect("CODEQL_DIST not set");
let db = env::var("CODEQL_EXTRACTOR_RUBY_WIP_DATABASE")
let database = env::var("CODEQL_EXTRACTOR_RUBY_WIP_DATABASE")
.expect("CODEQL_EXTRACTOR_RUBY_WIP_DATABASE not set");
let codeql = if env::consts::OS == "windows" {
"codeql.exe"
} else {
"codeql"
};
let codeql: PathBuf = [&dist, codeql].iter().collect();
let mut cmd = Command::new(codeql);
cmd.arg("database")
.arg("index-files")
.arg("--include-extension=.rb")
.arg("--include-extension=.erb")
.arg("--include-extension=.gemspec")
.arg("--include=**/Gemfile")
.arg("--exclude=**/.git")
.arg("--size-limit=5m")
.arg("--language=ruby")
.arg("--working-dir=.")
.arg(db);
for line in env::var("LGTM_INDEX_FILTERS")
.unwrap_or_default()
.split('\n')
{
if let Some(stripped) = line.strip_prefix("include:") {
cmd.arg("--also-match=".to_owned() + stripped);
} else if let Some(stripped) = line.strip_prefix("exclude:") {
cmd.arg("--exclude=".to_owned() + stripped);
}
}
let exit = &cmd.spawn()?.wait()?;
std::process::exit(exit.code().unwrap_or(1))
autobuilder::Autobuilder::new("ruby", PathBuf::from(database))
.include_extensions(&[".rb", ".erb", ".gemspec"])
.include_globs(&["**/Gemfile"])
.exclude_globs(&["**/.git"])
.size_limit("5m")
.run()
}

90
shared/tree-sitter-extractor/src/autobuilder.rs Normal file
View File

@@ -0,0 +1,90 @@
use std::env;
use std::path::PathBuf;
use std::process::Command;
pub struct Autobuilder {
include_extensions: Vec<String>,
include_globs: Vec<String>,
exclude_globs: Vec<String>,
language: String,
database: PathBuf,
size_limit: Option<String>,
}
impl Autobuilder {
pub fn new(language: &str, database: PathBuf) -> Self {
Self {
language: language.to_string(),
database: database,
include_extensions: vec![],
include_globs: vec![],
exclude_globs: vec![],
size_limit: None,
}
}
pub fn include_extensions(&mut self, exts: &[&str]) -> &mut Self {
self.include_extensions = exts.into_iter().map(|s| String::from(*s)).collect();
self
}
pub fn include_globs(&mut self, globs: &[&str]) -> &mut Self {
self.include_globs = globs.into_iter().map(|s| String::from(*s)).collect();
self
}
pub fn exclude_globs(&mut self, globs: &[&str]) -> &mut Self {
self.exclude_globs = globs.into_iter().map(|s| String::from(*s)).collect();
self
}
pub fn size_limit(&mut self, limit: &str) -> &mut Self {
self.size_limit = Some(limit.to_string());
self
}
pub fn run(&self) -> std::io::Result<()> {
let dist = env::var("CODEQL_DIST").expect("CODEQL_DIST not set");
let codeql = if env::consts::OS == "windows" {
"codeql.exe"
} else {
"codeql"
};
let codeql: PathBuf = [&dist, codeql].iter().collect();
let mut cmd = Command::new(codeql);
cmd.arg("database").arg("index-files");
for ext in &self.include_extensions {
cmd.arg(format!("--include-extension={}", ext));
}
for glob in &self.include_globs {
cmd.arg(format!("--include={}", glob));
}
for glob in &self.exclude_globs {
cmd.arg(format!("--exclude={}", glob));
}
if let Some(limit) = &self.size_limit {
cmd.arg(format!("--size-limit={}", limit));
}
cmd.arg(format!("--language={}", &self.language));
cmd.arg("--working-dir=.");
cmd.arg(&self.database);
for line in env::var("LGTM_INDEX_FILTERS")
.unwrap_or_default()
.split('\n')
{
if let Some(stripped) = line.strip_prefix("include:") {
cmd.arg("--also-match=".to_owned() + stripped);
} else if let Some(stripped) = line.strip_prefix("exclude:") {
cmd.arg("--exclude=".to_owned() + stripped);
}
}
let exit = &cmd.spawn()?.wait()?;
std::process::exit(exit.code().unwrap_or(1))
}
}

1
shared/tree-sitter-extractor/src/lib.rs
View File

@@ -1,3 +1,4 @@
pub mod autobuilder;
pub mod diagnostics;
pub mod extractor;
pub mod file_paths;