hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2025-12-16 16:53:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

Rust: move qltest to rust code, add options with cargo check

Browse Source
This commit is contained in:
Paolo Tranquilli
2024-10-24 11:51:13 +02:00
parent 226756e1d4
commit 3f8d438851
19 changed files with 263 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

97
Cargo.lock generated
View File

@@ -384,6 +384,8 @@ dependencies = [
"clap",
"codeql-extractor",
"figment",
"gag",
"glob",
"itertools 0.13.0",
"log",
"num-traits",
@@ -622,6 +624,22 @@ version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
[[package]]
name = "errno"
version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba"
dependencies = [
"libc",
"windows-sys 0.52.0",
]
[[package]]
name = "fastrand"
version = "2.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6"
[[package]]
name = "figment"
version = "0.10.19"
@@ -631,10 +649,22 @@ dependencies = [
"atomic",
"pear",
"serde",
"serde_yaml",
"uncased",
"version_check",
]
[[package]]
name = "filedescriptor"
version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7199d965852c3bac31f779ef99cbb4537f80e952e2d6aa0ffeb30cce00f4f46e"
dependencies = [
"libc",
"thiserror",
"winapi",
]
[[package]]
name = "filetime"
version = "0.2.25"
@@ -693,6 +723,16 @@ version = "0.4.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ab85b9b05e3978cc9a9cf8fea7f01b494e1a09ed3037e16ba39edc7a29eb61a"
[[package]]
name = "gag"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a713bee13966e9fbffdf7193af71d54a6b35a0bb34997cd6c9519ebeb5005972"
dependencies = [
"filedescriptor",
"tempfile",
]
[[package]]
name = "getrandom"
version = "0.2.15"
@@ -704,6 +744,12 @@ dependencies = [
"wasi",
]
[[package]]
name = "glob"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b"
[[package]]
name = "globset"
version = "0.4.15"
@@ -969,6 +1015,12 @@ dependencies = [
"text-size",
]
[[package]]
name = "linux-raw-sys"
version = "0.4.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89"
[[package]]
name = "lock_api"
version = "0.4.12"
@@ -1960,6 +2012,19 @@ dependencies = [
"smallvec",
]
[[package]]
name = "rustix"
version = "0.38.37"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8acb788b847c24f28525660c4d7758620a7210875711f79e7f663cc152726811"
dependencies = [
"bitflags 2.6.0",
"errno",
"libc",
"linux-raw-sys",
"windows-sys 0.52.0",
]
[[package]]
name = "ryu"
version = "1.0.18"
@@ -2058,6 +2123,19 @@ dependencies = [
"syn",
]
[[package]]
name = "serde_yaml"
version = "0.9.34+deprecated"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47"
dependencies = [
"indexmap 2.5.0",
"itoa",
"ryu",
"serde",
"unsafe-libyaml",
]
[[package]]
name = "sharded-slab"
version = "0.1.7"
@@ -2141,6 +2219,19 @@ dependencies = [
"syn",
]
[[package]]
name = "tempfile"
version = "3.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f0f2c9fc62d0beef6951ccffd757e241266a2c833136efbe35af6cd2567dca5b"
dependencies = [
"cfg-if",
"fastrand",
"once_cell",
"rustix",
"windows-sys 0.59.0",
]
[[package]]
name = "termcolor"
version = "1.1.3"
@@ -2381,6 +2472,12 @@ version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "229730647fbc343e3a80e463c1db7f78f3855d3f3739bee0dda773c9a037c90a"
[[package]]
name = "unsafe-libyaml"
version = "0.2.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861"
[[package]]
name = "utf8parse"
version = "0.2.2"

4
rust/extractor/Cargo.toml
View File

@@ -6,7 +6,7 @@ edition = "2021"
[dependencies]
anyhow = "1.0.86"
clap = { version = "4.5.16", features = ["derive"] }
figment = { version = "0.10.19", features = ["env"]}
figment = { version = "0.10.19", features = ["env", "yaml"] }
log = "0.4.22"
num-traits = "0.2.19"
ra_ap_base_db = "0.0.232"
@@ -29,3 +29,5 @@ argfile = "0.2.1"
codeql-extractor = { path = "../../shared/tree-sitter-extractor" }
rust-extractor-macros = { path = "macros" }
itertools = "0.13.0"
glob = "0.3.1"
gag = "1.0.0"

28
rust/extractor/src/config.rs
View File

@@ -2,9 +2,11 @@ use anyhow::Context;
use clap::Parser;
use codeql_extractor::trap;
use figment::{
providers::{Env, Serialized},
providers::{Env, Format, Serialized, Yaml},
value::Value,
Figment,
};
use itertools::Itertools;
use num_traits::Zero;
use rust_extractor_macros::extractor_cli_config;
use serde::{Deserialize, Serialize};
@@ -34,10 +36,13 @@ pub struct Config {
pub scratch_dir: PathBuf,
pub trap_dir: PathBuf,
pub source_archive_dir: PathBuf,
pub log_dir: PathBuf,
pub extract_dependencies: bool,
pub verbose: u8,
pub compression: Compression,
pub inputs: Vec<PathBuf>,
pub qltest: bool,
pub qltest_cargo_check: bool,
}
impl Config {
@@ -45,12 +50,25 @@ impl Config {
let args = argfile::expand_args(argfile::parse_fromfile, argfile::PREFIX)
.context("expanding parameter files")?;
let cli_args = CliConfig::parse_from(args);
Figment::new()
let mut figment = Figment::new()
.merge(Env::prefixed("CODEQL_"))
.merge(Env::prefixed("CODEQL_EXTRACTOR_RUST_"))
.merge(Env::prefixed("CODEQL_EXTRACTOR_RUST_OPTION_"))
.merge(Serialized::defaults(cli_args))
.extract()
.context("loading configuration")
.merge(Serialized::defaults(cli_args));
if matches!(figment.find_value("qltest"), Ok(Value::Bool(_, true))) {
let cwd = std::env::current_dir()?;
let mut option_files = cwd
.ancestors()
// only travel up while we're within the test pack
.take_while_inclusive(|p| !p.join("qlpack.yml").exists())
.map(|p| p.join("options"))
.filter(|p| p.exists())
.collect_vec();
option_files.reverse();
for path in option_files {
figment = figment.merge(Yaml::file_exact(path));
}
}
figment.extract().context("loading configuration")
}
}

43
rust/extractor/src/main.rs
View File

@@ -1,16 +1,19 @@
use anyhow::Context;
use archive::Archiver;
use log::info;
use ra_ap_ide_db::line_index::{LineCol, LineIndex};
use ra_ap_project_model::ProjectManifest;
use rust_analyzer::{ParseResult, RustAnalyzer};
use std::fs::File;
use std::io::{BufRead, BufReader};
use std::{
collections::HashMap,
path::{Path, PathBuf},
};
use anyhow::Context;
use archive::Archiver;
use ra_ap_ide_db::line_index::{LineCol, LineIndex};
use ra_ap_project_model::ProjectManifest;
use rust_analyzer::{ParseResult, RustAnalyzer};
mod archive;
mod config;
pub mod generated;
mod qltest;
mod rust_analyzer;
mod translate;
pub mod trap;
@@ -65,16 +68,20 @@ fn extract(
)
});
}
fn main() -> anyhow::Result<()> {
let cfg = config::Config::extract().context("failed to load configuration")?;
fn run_extractor(mut cfg: config::Config) -> anyhow::Result<()> {
stderrlog::new()
.module(module_path!())
.verbosity(1 + cfg.verbose as usize)
.verbosity(cfg.verbose as usize)
.init()?;
if cfg.qltest {
qltest::prepare(&mut cfg)?;
}
info!("configuration: {cfg:#?}\n");
let traps = trap::TrapFileProvider::new(&cfg).context("failed to set up trap files")?;
let archiver = archive::Archiver {
root: cfg.source_archive_dir,
root: cfg.source_archive_dir.clone(),
};
let files: Vec<PathBuf> = cfg
.inputs
@@ -118,3 +125,19 @@ fn main() -> anyhow::Result<()> {
Ok(())
}
fn main() -> anyhow::Result<()> {
let cfg = config::Config::extract().context("failed to load configuration")?;
let qltest = cfg.qltest;
let qltest_log = cfg.log_dir.join("qltest.log");
let result = std::panic::catch_unwind(|| run_extractor(cfg));
if qltest && matches!(result, Err(_) | Ok(Err(_))) {
// in case of failure, print out the full log
let log = File::open(qltest_log).context("opening qltest.log")?;
let reader = BufReader::new(log);
for line in reader.lines() {
println!("{}", line.context("reading qltest.log")?);
}
}
result.unwrap()
}

84
rust/extractor/src/qltest.rs Normal file
View File

@@ -0,0 +1,84 @@
use crate::config::Config;
use anyhow::Context;
use glob::glob;
use itertools::Itertools;
use log::info;
use std::fs;
use std::process::Command;
fn dump_lib() -> anyhow::Result<()> {
let path_iterator = glob("*.rs").context("globbing test sources")?;
let paths = path_iterator
.collect::<Result<Vec<_>, _>>()
.context("fetching test sources")?;
let lib = paths
.iter()
.filter(|p| !["lib.rs", "main.rs"].contains(&p.file_name().unwrap().to_str().unwrap()))
.map(|p| format!("mod {};", p.file_stem().unwrap().to_str().unwrap()))
.join("\n");
fs::write("lib.rs", lib).context("writing lib.rs")
}
fn dump_cargo_manifest() -> anyhow::Result<()> {
let mut manifest = String::from(
r#"[workspace]
[package]
name = "test"
version="0.0.1"
edition="2021"
[lib]
path="lib.rs"
"#,
);
if fs::exists("main.rs").context("checking existence of main.rs")? {
manifest.push_str(
r#"[[bin]]
name = "main"
path = "main.rs"
"#,
);
}
fs::write("Cargo.toml", manifest).context("writing Cargo.toml")
}
fn set_sources(config: &mut Config) -> anyhow::Result<()> {
let path_iterator = glob("*.rs").context("globbing test sources")?;
config.inputs = path_iterator
.collect::<Result<Vec<_>, _>>()
.context("fetching test sources")?;
Ok(())
}
fn redirect_output(config: &Config) -> anyhow::Result<()> {
let log_path = config.log_dir.join("qltest.log");
let log = fs::OpenOptions::new()
.append(true)
.create(true)
.open(&log_path)
.context("opening qltest.log")?;
Box::leak(Box::new(
gag::Redirect::stderr(log).context("redirecting stderr")?,
));
Ok(())
}
pub(crate) fn prepare(config: &mut Config) -> anyhow::Result<()> {
redirect_output(config)?;
dump_lib()?;
set_sources(config)?;
dump_cargo_manifest()?;
if config.qltest_cargo_check {
let status = Command::new("cargo")
.env("RUSTFLAGS", "-Awarnings")
.arg("check")
.arg("-q")
.status()
.context("spawning cargo check")?;
if status.success() {
info!("cargo check successful");
} else {
anyhow::bail!("requested cargo check failed");
}
}
Ok(())
}

2
rust/ql/test/extractor-tests/generated/Module/Module.expected
View File

@@ -1,3 +1,3 @@
| gen_module.rs:3:1:4:8 | Module | getNumberOfAttrs: | 0 | hasItemList: | no | hasName: | yes | hasVisibility: | no |
| gen_module.rs:5:1:7:1 | Module | getNumberOfAttrs: | 0 | hasItemList: | yes | hasName: | yes | hasVisibility: | no |
| lib.rs:2:1:2:15 | Module | getNumberOfAttrs: | 0 | hasItemList: | no | hasName: | yes | hasVisibility: | no |
| lib.rs:1:1:1:15 | Module | getNumberOfAttrs: | 0 | hasItemList: | no | hasName: | yes | hasVisibility: | no |

2
rust/ql/test/extractor-tests/generated/Module/Module_getName.expected
View File

@@ -1,3 +1,3 @@
| gen_module.rs:3:1:4:8 | Module | gen_module.rs:4:5:4:7 | Name |
| gen_module.rs:5:1:7:1 | Module | gen_module.rs:5:5:5:7 | Name |
| lib.rs:2:1:2:15 | Module | lib.rs:2:5:2:14 | Name |
| lib.rs:1:1:1:15 | Module | lib.rs:1:5:1:14 | Name |

2
rust/ql/test/extractor-tests/generated/Name/Name.expected
View File

@@ -1,2 +1,2 @@
| gen_name.rs:3:4:3:12 | Name | hasText: | yes |
| lib.rs:2:5:2:12 | Name | hasText: | yes |
| lib.rs:1:5:1:12 | Name | hasText: | yes |

2
rust/ql/test/extractor-tests/generated/Name/Name_getText.expected
View File

@@ -1,2 +1,2 @@
| gen_name.rs:3:4:3:12 | Name | test_name |
| lib.rs:2:5:2:12 | Name | gen_name |
| lib.rs:1:5:1:12 | Name | gen_name |

2
rust/ql/test/extractor-tests/generated/SourceFile/SourceFile.expected
View File

@@ -1,2 +1,2 @@
| gen_source_file.rs:1:1:6:2 | SourceFile | getNumberOfAttrs: | 0 | getNumberOfItems: | 1 |
| lib.rs:1:1:2:21 | SourceFile | getNumberOfAttrs: | 0 | getNumberOfItems: | 1 |
| lib.rs:1:1:1:20 | SourceFile | getNumberOfAttrs: | 0 | getNumberOfItems: | 1 |

2
rust/ql/test/extractor-tests/generated/SourceFile/SourceFile_getItem.expected
View File

@@ -1,2 +1,2 @@
| gen_source_file.rs:1:1:6:2 | SourceFile | 0 | gen_source_file.rs:3:1:6:1 | test_source_file |
| lib.rs:1:1:2:21 | SourceFile | 0 | lib.rs:2:1:2:20 | Module |
| lib.rs:1:1:1:20 | SourceFile | 0 | lib.rs:1:1:1:20 | Module |

1
rust/ql/test/extractor-tests/generated/options Normal file
View File

@@ -0,0 +1 @@
qltest_cargo_check: false

6
rust/ql/test/extractor-tests/utf8/ast.expected
View File

@@ -1,6 +1,6 @@
| lib.rs:1:1:2:22 | SourceFile |
| lib.rs:2:1:2:21 | Module |
| lib.rs:2:5:2:20 | Name |
| lib.rs:1:1:1:21 | Module |
| lib.rs:1:1:1:21 | SourceFile |
| lib.rs:1:5:1:20 | Name |
| utf8_identifiers.rs:1:1:4:6 | foo |
| utf8_identifiers.rs:1:1:12:2 | SourceFile |
| utf8_identifiers.rs:1:4:1:6 | Name |

1
rust/ql/test/library-tests/controlflow/options Normal file
View File

@@ -0,0 +1 @@
qltest_cargo_check: false

1
rust/ql/test/library-tests/variables/options Normal file
View File

@@ -0,0 +1 @@
qltest_cargo_check: true

1
rust/ql/test/options Normal file
View File

@@ -0,0 +1 @@
qltest_cargo_check: true

1
rust/ql/test/query-tests/options Normal file
View File

@@ -0,0 +1 @@
qltest_cargo_check: false

7
rust/tools/qltest.cmd Normal file
View File

@@ -0,0 +1,7 @@
@echo off
set "RUST_BACKTRACE=full"
type NUL && "%CODEQL_EXTRACTOR_RUST_ROOT%/tools/%CODEQL_PLATFORM%/extractor" --qltest
exit /b %ERRORLEVEL%

37
rust/tools/qltest.sh
View File

@@ -1,40 +1,7 @@
#!/bin/bash
mkdir -p "$CODEQL_EXTRACTOR_RUST_TRAP_DIR"
set -eu
export RUST_BACKTRACE=full
QLTEST_LOG="$CODEQL_EXTRACTOR_RUST_LOG_DIR"/qltest.log
EXTRACTOR="$CODEQL_EXTRACTOR_RUST_ROOT/tools/$CODEQL_PLATFORM/extractor"
echo > lib.rs
for src in *.rs; do
if [[ "$src" == "lib.rs" ]]; then
continue
elif [[ "$src" == "main.rs" ]]; then
continue
else
echo "mod ${src%.rs};" >> lib.rs
fi
done
cat > Cargo.toml << EOF
[workspace]
[package]
name = "test"
version="0.0.1"
edition="2021"
[lib]
path="lib.rs"
EOF
if [[ -f "main.rs" ]]; then
cat >> Cargo.toml << EOF
[[bin]]
name = "main"
path = "main.rs"
EOF
fi
"$EXTRACTOR" *.rs >> "$QLTEST_LOG"
if [[ "$?" != 0 ]]; then
cat "$QLTEST_LOG" # Show compiler errors on extraction failure
exit 1
fi
"$CODEQL_EXTRACTOR_RUST_ROOT/tools/$CODEQL_PLATFORM/extractor" --qltest