hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2026-04-20 06:24:03 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #17835 from github/redsun82/rust-qltest

Browse Source 
Rust: move `qltest` to rust code, add `options` with cargo check
This commit is contained in:
Paolo Tranquilli
2024-10-24 18:29:17 +02:00
committed by GitHub
parent e920a4c56c 41d0085918
commit 55d092fd3e
20 changed files with 198 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
Cargo.lock generated
View File

@@ -384,6 +384,7 @@ dependencies = [
"clap",
"codeql-extractor",
"figment",
"glob",
"itertools 0.13.0",
"log",
"num-traits",
@@ -631,6 +632,7 @@ dependencies = [
"atomic",
"pear",
"serde",
"serde_yaml",
"uncased",
"version_check",
]
@@ -704,6 +706,12 @@ dependencies = [
"wasi",
]
[[package]]
name = "glob"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b"
[[package]]
name = "globset"
version = "0.4.15"
@@ -2058,6 +2066,19 @@ dependencies = [
"syn",
]
[[package]]
name = "serde_yaml"
version = "0.9.34+deprecated"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47"
dependencies = [
"indexmap 2.5.0",
"itoa",
"ryu",
"serde",
"unsafe-libyaml",
]
[[package]]
name = "sharded-slab"
version = "0.1.7"
@@ -2381,6 +2402,12 @@ version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "229730647fbc343e3a80e463c1db7f78f3855d3f3739bee0dda773c9a037c90a"
[[package]]
name = "unsafe-libyaml"
version = "0.2.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861"
[[package]]
name = "utf8parse"
version = "0.2.2"

3
rust/extractor/Cargo.toml
View File

@@ -6,7 +6,7 @@ edition = "2021"
[dependencies]
anyhow = "1.0.86"
clap = { version = "4.5.16", features = ["derive"] }
figment = { version = "0.10.19", features = ["env"]}
figment = { version = "0.10.19", features = ["env", "yaml"] }
log = "0.4.22"
num-traits = "0.2.19"
ra_ap_base_db = "0.0.232"
@@ -29,3 +29,4 @@ argfile = "0.2.1"
codeql-extractor = { path = "../../shared/tree-sitter-extractor" }
rust-extractor-macros = { path = "macros" }
itertools = "0.13.0"
glob = "0.3.1"

37
rust/extractor/macros/src/lib.rs
View File

@@ -6,8 +6,8 @@ use quote::{format_ident, quote};
pub fn extractor_cli_config(_attr: TokenStream, item: TokenStream) -> TokenStream {
let ast = syn::parse_macro_input!(item as syn::ItemStruct);
let name = &ast.ident;
let new_name = format_ident!("Cli{}", name);
let fields: Vec<_> = ast
let cli_name = format_ident!("Cli{}", name);
let cli_fields = ast
.fields
.iter()
.map(|f| {
@@ -39,17 +39,42 @@ pub fn extractor_cli_config(_attr: TokenStream, item: TokenStream) -> TokenStrea
}
}
})
.collect();
.collect::<Vec<_>>();
let debug_fields = ast
.fields
.iter()
.map(|f| {
let id = f.ident.as_ref().unwrap();
if id == &format_ident!("inputs") {
quote! {
.field("number of inputs", &self.#id.len())
}
} else {
quote! {
.field(stringify!(#id), &self.#id)
}
}
})
.collect::<Vec<_>>();
let gen = quote! {
#[serde_with::apply(_ => #[serde(default)])]
#[derive(Debug, Deserialize, Default)]
#[derive(Deserialize, Default)]
#ast
impl Debug for #name {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("configuration:")
#(#debug_fields)*
.finish()
}
}
#[serde_with::skip_serializing_none]
#[derive(clap::Parser, Serialize)]
#[command(about, long_about = None)]
struct #new_name {
#(#fields)*
struct #cli_name {
#(#cli_fields)*
}
};
gen.into()

28
rust/extractor/src/config.rs
View File

@@ -2,12 +2,15 @@ use anyhow::Context;
use clap::Parser;
use codeql_extractor::trap;
use figment::{
providers::{Env, Serialized},
providers::{Env, Format, Serialized, Yaml},
value::Value,
Figment,
};
use itertools::Itertools;
use num_traits::Zero;
use rust_extractor_macros::extractor_cli_config;
use serde::{Deserialize, Serialize};
use std::fmt::Debug;
use std::ops::Not;
use std::path::PathBuf;
@@ -38,6 +41,8 @@ pub struct Config {
pub verbose: u8,
pub compression: Compression,
pub inputs: Vec<PathBuf>,
pub qltest: bool,
pub qltest_cargo_check: bool,
}
impl Config {
@@ -45,12 +50,25 @@ impl Config {
let args = argfile::expand_args(argfile::parse_fromfile, argfile::PREFIX)
.context("expanding parameter files")?;
let cli_args = CliConfig::parse_from(args);
Figment::new()
let mut figment = Figment::new()
.merge(Env::prefixed("CODEQL_"))
.merge(Env::prefixed("CODEQL_EXTRACTOR_RUST_"))
.merge(Env::prefixed("CODEQL_EXTRACTOR_RUST_OPTION_"))
.merge(Serialized::defaults(cli_args))
.extract()
.context("loading configuration")
.merge(Serialized::defaults(cli_args));
if let Ok(Value::Bool(_, true)) = figment.find_value("qltest") {
let cwd = std::env::current_dir()?;
let mut option_files = cwd
.ancestors()
// only travel up while we're within the test pack
.take_while_inclusive(|p| !p.join("qlpack.yml").exists())
.map(|p| p.join("options"))
.filter(|p| p.exists())
.collect_vec();
option_files.reverse();
for path in option_files {
figment = figment.merge(Yaml::file_exact(path));
}
}
figment.extract().context("loading configuration")
}
}

24
rust/extractor/src/main.rs
View File

@@ -1,16 +1,17 @@
use anyhow::Context;
use archive::Archiver;
use log::info;
use ra_ap_ide_db::line_index::{LineCol, LineIndex};
use ra_ap_project_model::ProjectManifest;
use rust_analyzer::{ParseResult, RustAnalyzer};
use std::{
collections::HashMap,
path::{Path, PathBuf},
};
use anyhow::Context;
use archive::Archiver;
use ra_ap_ide_db::line_index::{LineCol, LineIndex};
use ra_ap_project_model::ProjectManifest;
use rust_analyzer::{ParseResult, RustAnalyzer};
mod archive;
mod config;
pub mod generated;
mod qltest;
mod rust_analyzer;
mod translate;
pub mod trap;
@@ -65,16 +66,21 @@ fn extract(
)
});
}
fn main() -> anyhow::Result<()> {
let cfg = config::Config::extract().context("failed to load configuration")?;
let mut cfg = config::Config::extract().context("failed to load configuration")?;
stderrlog::new()
.module(module_path!())
.verbosity(1 + cfg.verbose as usize)
.verbosity(2 + cfg.verbose as usize)
.init()?;
if cfg.qltest {
qltest::prepare(&mut cfg)?;
}
info!("{cfg:#?}\n");
let traps = trap::TrapFileProvider::new(&cfg).context("failed to set up trap files")?;
let archiver = archive::Archiver {
root: cfg.source_archive_dir,
root: cfg.source_archive_dir.clone(),
};
let files: Vec<PathBuf> = cfg
.inputs

72
rust/extractor/src/qltest.rs Normal file
View File

@@ -0,0 +1,72 @@
use crate::config::Config;
use anyhow::Context;
use glob::glob;
use itertools::Itertools;
use log::info;
use std::ffi::OsStr;
use std::fs;
use std::process::Command;
fn dump_lib() -> anyhow::Result<()> {
let path_iterator = glob("*.rs").context("globbing test sources")?;
let paths = path_iterator
.collect::<Result<Vec<_>, _>>()
.context("fetching test sources")?;
let lib = paths
.iter()
.map(|p| p.file_stem().expect("results of glob must have a name"))
.filter(|&p| !["main", "lib"].map(OsStr::new).contains(&p))
.map(|p| format!("mod {};", p.to_string_lossy()))
.join("\n");
fs::write("lib.rs", lib).context("writing lib.rs")
}
fn dump_cargo_manifest() -> anyhow::Result<()> {
let mut manifest = String::from(
r#"[workspace]
[package]
name = "test"
version="0.0.1"
edition="2021"
[lib]
path="lib.rs"
"#,
);
if fs::exists("main.rs").context("checking existence of main.rs")? {
manifest.push_str(
r#"[[bin]]
name = "main"
path = "main.rs"
"#,
);
}
fs::write("Cargo.toml", manifest).context("writing Cargo.toml")
}
fn set_sources(config: &mut Config) -> anyhow::Result<()> {
let path_iterator = glob("*.rs").context("globbing test sources")?;
config.inputs = path_iterator
.collect::<Result<Vec<_>, _>>()
.context("fetching test sources")?;
Ok(())
}
pub(crate) fn prepare(config: &mut Config) -> anyhow::Result<()> {
dump_lib()?;
set_sources(config)?;
dump_cargo_manifest()?;
if config.qltest_cargo_check {
let status = Command::new("cargo")
.env("RUSTFLAGS", "-Awarnings")
.arg("check")
.arg("-q")
.status()
.context("spawning cargo check")?;
if status.success() {
info!("cargo check successful");
} else {
anyhow::bail!("requested cargo check failed");
}
}
Ok(())
}

2
rust/ql/test/extractor-tests/generated/Module/Module.expected
View File

@@ -1,3 +1,3 @@
| gen_module.rs:3:1:4:8 | Module | getNumberOfAttrs: | 0 | hasItemList: | no | hasName: | yes | hasVisibility: | no |
| gen_module.rs:5:1:7:1 | Module | getNumberOfAttrs: | 0 | hasItemList: | yes | hasName: | yes | hasVisibility: | no |
| lib.rs:2:1:2:15 | Module | getNumberOfAttrs: | 0 | hasItemList: | no | hasName: | yes | hasVisibility: | no |
| lib.rs:1:1:1:15 | Module | getNumberOfAttrs: | 0 | hasItemList: | no | hasName: | yes | hasVisibility: | no |

2
rust/ql/test/extractor-tests/generated/Module/Module_getName.expected
View File

@@ -1,3 +1,3 @@
| gen_module.rs:3:1:4:8 | Module | gen_module.rs:4:5:4:7 | Name |
| gen_module.rs:5:1:7:1 | Module | gen_module.rs:5:5:5:7 | Name |
| lib.rs:2:1:2:15 | Module | lib.rs:2:5:2:14 | Name |
| lib.rs:1:1:1:15 | Module | lib.rs:1:5:1:14 | Name |

2
rust/ql/test/extractor-tests/generated/Name/Name.expected
View File

@@ -1,2 +1,2 @@
| gen_name.rs:3:4:3:12 | Name | hasText: | yes |
| lib.rs:2:5:2:12 | Name | hasText: | yes |
| lib.rs:1:5:1:12 | Name | hasText: | yes |

2
rust/ql/test/extractor-tests/generated/Name/Name_getText.expected
View File

@@ -1,2 +1,2 @@
| gen_name.rs:3:4:3:12 | Name | test_name |
| lib.rs:2:5:2:12 | Name | gen_name |
| lib.rs:1:5:1:12 | Name | gen_name |

2
rust/ql/test/extractor-tests/generated/SourceFile/SourceFile.expected
View File

@@ -1,2 +1,2 @@
| gen_source_file.rs:1:1:6:2 | SourceFile | getNumberOfAttrs: | 0 | getNumberOfItems: | 1 |
| lib.rs:1:1:2:21 | SourceFile | getNumberOfAttrs: | 0 | getNumberOfItems: | 1 |
| lib.rs:1:1:1:20 | SourceFile | getNumberOfAttrs: | 0 | getNumberOfItems: | 1 |

2
rust/ql/test/extractor-tests/generated/SourceFile/SourceFile_getItem.expected
View File

@@ -1,2 +1,2 @@
| gen_source_file.rs:1:1:6:2 | SourceFile | 0 | gen_source_file.rs:3:1:6:1 | test_source_file |
| lib.rs:1:1:2:21 | SourceFile | 0 | lib.rs:2:1:2:20 | Module |
| lib.rs:1:1:1:20 | SourceFile | 0 | lib.rs:1:1:1:20 | Module |

1
rust/ql/test/extractor-tests/generated/options Normal file
View File

@@ -0,0 +1 @@
qltest_cargo_check: false

6
rust/ql/test/extractor-tests/utf8/ast.expected
View File

@@ -1,6 +1,6 @@
| lib.rs:1:1:2:22 | SourceFile |
| lib.rs:2:1:2:21 | Module |
| lib.rs:2:5:2:20 | Name |
| lib.rs:1:1:1:21 | Module |
| lib.rs:1:1:1:21 | SourceFile |
| lib.rs:1:5:1:20 | Name |
| utf8_identifiers.rs:1:1:4:6 | foo |
| utf8_identifiers.rs:1:1:12:2 | SourceFile |
| utf8_identifiers.rs:1:4:1:6 | Name |

1
rust/ql/test/library-tests/controlflow/options Normal file
View File

@@ -0,0 +1 @@
qltest_cargo_check: false

1
rust/ql/test/options Normal file
View File

@@ -0,0 +1 @@
qltest_cargo_check: true

1
rust/ql/test/query-tests/diagnostics/options Normal file
View File

@@ -0,0 +1 @@
qltest_cargo_check: false

1
rust/ql/test/query-tests/unusedentities/options Normal file
View File

@@ -0,0 +1 @@
qltest_cargo_check: false

11
rust/tools/qltest.cmd Normal file
View File

@@ -0,0 +1,11 @@
@echo off
set "RUST_BACKTRACE=full"
set "QLTEST_LOG=%CODEQL_EXTRACTOR_RUST_LOG_DIR%/qltest.log"
type NUL && "%CODEQL_EXTRACTOR_RUST_ROOT%/tools/%CODEQL_PLATFORM%/extractor" --qltest >"%QLTEST_LOG%"
if %ERRORLEVEL% neq 0 (
type "%QLTEST_LOG%"
exit /b 1
)

36
rust/tools/qltest.sh
View File

@@ -1,40 +1,10 @@
#!/bin/bash
mkdir -p "$CODEQL_EXTRACTOR_RUST_TRAP_DIR"
set -eu
export RUST_BACKTRACE=full
QLTEST_LOG="$CODEQL_EXTRACTOR_RUST_LOG_DIR"/qltest.log
EXTRACTOR="$CODEQL_EXTRACTOR_RUST_ROOT/tools/$CODEQL_PLATFORM/extractor"
echo > lib.rs
for src in *.rs; do
if [[ "$src" == "lib.rs" ]]; then
continue
elif [[ "$src" == "main.rs" ]]; then
continue
else
echo "mod ${src%.rs};" >> lib.rs
fi
done
cat > Cargo.toml << EOF
[workspace]
[package]
name = "test"
version="0.0.1"
edition="2021"
[lib]
path="lib.rs"
EOF
if [[ -f "main.rs" ]]; then
cat >> Cargo.toml << EOF
[[bin]]
name = "main"
path = "main.rs"
EOF
fi
"$EXTRACTOR" *.rs >> "$QLTEST_LOG"
if [[ "$?" != 0 ]]; then
cat "$QLTEST_LOG" # Show compiler errors on extraction failure
if ! "$CODEQL_EXTRACTOR_RUST_ROOT/tools/$CODEQL_PLATFORM/extractor" --qltest &>> "$QLTEST_LOG"; then
cat "$QLTEST_LOG"
exit 1
fi