Rust: move qltest to rust code, add options with cargo check

Cargo.lock

@@ -384,6 +384,8 @@ dependencies = [
  "clap",
  "codeql-extractor",
  "figment",
+ "gag",
+ "glob",
  "itertools 0.13.0",
  "log",
  "num-traits",
@@ -622,6 +624,22 @@ version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
 
+[[package]]
+name = "errno"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba"
+dependencies = [
+ "libc",
+ "windows-sys 0.52.0",
+]
+
+[[package]]
+name = "fastrand"
+version = "2.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6"
+
 [[package]]
 name = "figment"
 version = "0.10.19"
@@ -631,10 +649,22 @@ dependencies = [
  "atomic",
  "pear",
  "serde",
+ "serde_yaml",
  "uncased",
  "version_check",
 ]
 
+[[package]]
+name = "filedescriptor"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7199d965852c3bac31f779ef99cbb4537f80e952e2d6aa0ffeb30cce00f4f46e"
+dependencies = [
+ "libc",
+ "thiserror",
+ "winapi",
+]
+
 [[package]]
 name = "filetime"
 version = "0.2.25"
@@ -693,6 +723,16 @@ version = "0.4.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7ab85b9b05e3978cc9a9cf8fea7f01b494e1a09ed3037e16ba39edc7a29eb61a"
 
+[[package]]
+name = "gag"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a713bee13966e9fbffdf7193af71d54a6b35a0bb34997cd6c9519ebeb5005972"
+dependencies = [
+ "filedescriptor",
+ "tempfile",
+]
+
 [[package]]
 name = "getrandom"
 version = "0.2.15"
@@ -704,6 +744,12 @@ dependencies = [
  "wasi",
 ]
 
+[[package]]
+name = "glob"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b"
+
 [[package]]
 name = "globset"
 version = "0.4.15"
@@ -969,6 +1015,12 @@ dependencies = [
  "text-size",
 ]
 
+[[package]]
+name = "linux-raw-sys"
+version = "0.4.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89"
+
 [[package]]
 name = "lock_api"
 version = "0.4.12"
@@ -1960,6 +2012,19 @@ dependencies = [
  "smallvec",
 ]
 
+[[package]]
+name = "rustix"
+version = "0.38.37"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8acb788b847c24f28525660c4d7758620a7210875711f79e7f663cc152726811"
+dependencies = [
+ "bitflags 2.6.0",
+ "errno",
+ "libc",
+ "linux-raw-sys",
+ "windows-sys 0.52.0",
+]
+
 [[package]]
 name = "ryu"
 version = "1.0.18"
@@ -2058,6 +2123,19 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "serde_yaml"
+version = "0.9.34+deprecated"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47"
+dependencies = [
+ "indexmap 2.5.0",
+ "itoa",
+ "ryu",
+ "serde",
+ "unsafe-libyaml",
+]
+
 [[package]]
 name = "sharded-slab"
 version = "0.1.7"
@@ -2141,6 +2219,19 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "tempfile"
+version = "3.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f0f2c9fc62d0beef6951ccffd757e241266a2c833136efbe35af6cd2567dca5b"
+dependencies = [
+ "cfg-if",
+ "fastrand",
+ "once_cell",
+ "rustix",
+ "windows-sys 0.59.0",
+]
+
 [[package]]
 name = "termcolor"
 version = "1.1.3"
@@ -2381,6 +2472,12 @@ version = "0.2.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "229730647fbc343e3a80e463c1db7f78f3855d3f3739bee0dda773c9a037c90a"
 
+[[package]]
+name = "unsafe-libyaml"
+version = "0.2.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861"
+
 [[package]]
 name = "utf8parse"
 version = "0.2.2"

rust/extractor/Cargo.toml

@@ -6,7 +6,7 @@ edition = "2021"
 [dependencies]
 anyhow = "1.0.86"
 clap = { version = "4.5.16", features = ["derive"] }
-figment = { version = "0.10.19", features = ["env"]}
+figment = { version = "0.10.19", features = ["env", "yaml"] }
 log = "0.4.22"
 num-traits = "0.2.19"
 ra_ap_base_db = "0.0.232"
@@ -29,3 +29,5 @@ argfile = "0.2.1"
 codeql-extractor = { path = "../../shared/tree-sitter-extractor" }
 rust-extractor-macros = { path = "macros" }
 itertools = "0.13.0"
+glob = "0.3.1"
+gag = "1.0.0"

rust/extractor/src/config.rs

@@ -2,9 +2,11 @@ use anyhow::Context;
 use clap::Parser;
 use codeql_extractor::trap;
 use figment::{
-    providers::{Env, Serialized},
+    providers::{Env, Format, Serialized, Yaml},
+    value::Value,
     Figment,
 };
+use itertools::Itertools;
 use num_traits::Zero;
 use rust_extractor_macros::extractor_cli_config;
 use serde::{Deserialize, Serialize};
@@ -34,10 +36,13 @@ pub struct Config {
     pub scratch_dir: PathBuf,
     pub trap_dir: PathBuf,
     pub source_archive_dir: PathBuf,
+    pub log_dir: PathBuf,
     pub extract_dependencies: bool,
     pub verbose: u8,
     pub compression: Compression,
     pub inputs: Vec<PathBuf>,
+    pub qltest: bool,
+    pub qltest_cargo_check: bool,
 }
 
 impl Config {
@@ -45,12 +50,25 @@ impl Config {
         let args = argfile::expand_args(argfile::parse_fromfile, argfile::PREFIX)
             .context("expanding parameter files")?;
         let cli_args = CliConfig::parse_from(args);
-        Figment::new()
+        let mut figment = Figment::new()
             .merge(Env::prefixed("CODEQL_"))
             .merge(Env::prefixed("CODEQL_EXTRACTOR_RUST_"))
             .merge(Env::prefixed("CODEQL_EXTRACTOR_RUST_OPTION_"))
-            .merge(Serialized::defaults(cli_args))
+            .merge(Serialized::defaults(cli_args));
-            .extract()
+        if matches!(figment.find_value("qltest"), Ok(Value::Bool(_, true))) {
-            .context("loading configuration")
+            let cwd = std::env::current_dir()?;
+            let mut option_files = cwd
+                .ancestors()
+                // only travel up while we're within the test pack
+                .take_while_inclusive(|p| !p.join("qlpack.yml").exists())
+                .map(|p| p.join("options"))
+                .filter(|p| p.exists())
+                .collect_vec();
+            option_files.reverse();
+            for path in option_files {
+                figment = figment.merge(Yaml::file_exact(path));
+            }
+        }
+        figment.extract().context("loading configuration")
     }
 }

rust/extractor/src/main.rs

@@ -1,16 +1,19 @@
+use anyhow::Context;
+use archive::Archiver;
+use log::info;
+use ra_ap_ide_db::line_index::{LineCol, LineIndex};
+use ra_ap_project_model::ProjectManifest;
+use rust_analyzer::{ParseResult, RustAnalyzer};
+use std::fs::File;
+use std::io::{BufRead, BufReader};
 use std::{
     collections::HashMap,
     path::{Path, PathBuf},
 };
-
-use anyhow::Context;
-use archive::Archiver;
-use ra_ap_ide_db::line_index::{LineCol, LineIndex};
-use ra_ap_project_model::ProjectManifest;
-use rust_analyzer::{ParseResult, RustAnalyzer};
 mod archive;
 mod config;
 pub mod generated;
+mod qltest;
 mod rust_analyzer;
 mod translate;
 pub mod trap;
@@ -65,16 +68,20 @@ fn extract(
         )
     });
 }
-fn main() -> anyhow::Result<()> {
+
-    let cfg = config::Config::extract().context("failed to load configuration")?;
+fn run_extractor(mut cfg: config::Config) -> anyhow::Result<()> {
     stderrlog::new()
         .module(module_path!())
-        .verbosity(1 + cfg.verbose as usize)
+        .verbosity(cfg.verbose as usize)
         .init()?;
+    if cfg.qltest {
+        qltest::prepare(&mut cfg)?;
+    }
+    info!("configuration: {cfg:#?}\n");
 
     let traps = trap::TrapFileProvider::new(&cfg).context("failed to set up trap files")?;
     let archiver = archive::Archiver {
-        root: cfg.source_archive_dir,
+        root: cfg.source_archive_dir.clone(),
     };
     let files: Vec<PathBuf> = cfg
         .inputs
@@ -118,3 +125,19 @@ fn main() -> anyhow::Result<()> {
 
     Ok(())
 }
+
+fn main() -> anyhow::Result<()> {
+    let cfg = config::Config::extract().context("failed to load configuration")?;
+    let qltest = cfg.qltest;
+    let qltest_log = cfg.log_dir.join("qltest.log");
+    let result = std::panic::catch_unwind(|| run_extractor(cfg));
+    if qltest && matches!(result, Err(_) | Ok(Err(_))) {
+        // in case of failure, print out the full log
+        let log = File::open(qltest_log).context("opening qltest.log")?;
+        let reader = BufReader::new(log);
+        for line in reader.lines() {
+            println!("{}", line.context("reading qltest.log")?);
+        }
+    }
+    result.unwrap()
+}

rust/extractor/src/qltest.rs

@@ -0,0 +1,84 @@
+use crate::config::Config;
+use anyhow::Context;
+use glob::glob;
+use itertools::Itertools;
+use log::info;
+use std::fs;
+use std::process::Command;
+
+fn dump_lib() -> anyhow::Result<()> {
+    let path_iterator = glob("*.rs").context("globbing test sources")?;
+    let paths = path_iterator
+        .collect::<Result<Vec<_>, _>>()
+        .context("fetching test sources")?;
+    let lib = paths
+        .iter()
+        .filter(|p| !["lib.rs", "main.rs"].contains(&p.file_name().unwrap().to_str().unwrap()))
+        .map(|p| format!("mod {};", p.file_stem().unwrap().to_str().unwrap()))
+        .join("\n");
+    fs::write("lib.rs", lib).context("writing lib.rs")
+}
+
+fn dump_cargo_manifest() -> anyhow::Result<()> {
+    let mut manifest = String::from(
+        r#"[workspace]
+[package]
+name = "test"
+version="0.0.1"
+edition="2021"
+[lib]
+path="lib.rs"
+"#,
+    );
+    if fs::exists("main.rs").context("checking existence of main.rs")? {
+        manifest.push_str(
+            r#"[[bin]]
+name = "main"
+path = "main.rs"
+"#,
+        );
+    }
+    fs::write("Cargo.toml", manifest).context("writing Cargo.toml")
+}
+
+fn set_sources(config: &mut Config) -> anyhow::Result<()> {
+    let path_iterator = glob("*.rs").context("globbing test sources")?;
+    config.inputs = path_iterator
+        .collect::<Result<Vec<_>, _>>()
+        .context("fetching test sources")?;
+    Ok(())
+}
+
+fn redirect_output(config: &Config) -> anyhow::Result<()> {
+    let log_path = config.log_dir.join("qltest.log");
+    let log = fs::OpenOptions::new()
+        .append(true)
+        .create(true)
+        .open(&log_path)
+        .context("opening qltest.log")?;
+    Box::leak(Box::new(
+        gag::Redirect::stderr(log).context("redirecting stderr")?,
+    ));
+    Ok(())
+}
+
+pub(crate) fn prepare(config: &mut Config) -> anyhow::Result<()> {
+    redirect_output(config)?;
+    dump_lib()?;
+    set_sources(config)?;
+    dump_cargo_manifest()?;
+    if config.qltest_cargo_check {
+        let status = Command::new("cargo")
+            .env("RUSTFLAGS", "-Awarnings")
+            .arg("check")
+            .arg("-q")
+            .status()
+            .context("spawning cargo check")?;
+        if status.success() {
+            info!("cargo check successful");
+        } else {
+            anyhow::bail!("requested cargo check failed");
+        }
+    }
+    Ok(())
+}

rust/ql/test/extractor-tests/generated/Module/Module.expected

@@ -1,3 +1,3 @@
 | gen_module.rs:3:1:4:8 | Module | getNumberOfAttrs: | 0 | hasItemList: | no | hasName: | yes | hasVisibility: | no |
 | gen_module.rs:5:1:7:1 | Module | getNumberOfAttrs: | 0 | hasItemList: | yes | hasName: | yes | hasVisibility: | no |
-| lib.rs:2:1:2:15 | Module | getNumberOfAttrs: | 0 | hasItemList: | no | hasName: | yes | hasVisibility: | no |
+| lib.rs:1:1:1:15 | Module | getNumberOfAttrs: | 0 | hasItemList: | no | hasName: | yes | hasVisibility: | no |

rust/ql/test/extractor-tests/generated/Module/Module_getName.expected

@@ -1,3 +1,3 @@
 | gen_module.rs:3:1:4:8 | Module | gen_module.rs:4:5:4:7 | Name |
 | gen_module.rs:5:1:7:1 | Module | gen_module.rs:5:5:5:7 | Name |
-| lib.rs:2:1:2:15 | Module | lib.rs:2:5:2:14 | Name |
+| lib.rs:1:1:1:15 | Module | lib.rs:1:5:1:14 | Name |

rust/ql/test/extractor-tests/generated/Name/Name.expected

@@ -1,2 +1,2 @@
 | gen_name.rs:3:4:3:12 | Name | hasText: | yes |
-| lib.rs:2:5:2:12 | Name | hasText: | yes |
+| lib.rs:1:5:1:12 | Name | hasText: | yes |

rust/ql/test/extractor-tests/generated/Name/Name_getText.expected

@@ -1,2 +1,2 @@
 | gen_name.rs:3:4:3:12 | Name | test_name |
-| lib.rs:2:5:2:12 | Name | gen_name |
+| lib.rs:1:5:1:12 | Name | gen_name |

rust/ql/test/extractor-tests/generated/SourceFile/SourceFile.expected

@@ -1,2 +1,2 @@
 | gen_source_file.rs:1:1:6:2 | SourceFile | getNumberOfAttrs: | 0 | getNumberOfItems: | 1 |
-| lib.rs:1:1:2:21 | SourceFile | getNumberOfAttrs: | 0 | getNumberOfItems: | 1 |
+| lib.rs:1:1:1:20 | SourceFile | getNumberOfAttrs: | 0 | getNumberOfItems: | 1 |

rust/ql/test/extractor-tests/generated/SourceFile/SourceFile_getItem.expected

@@ -1,2 +1,2 @@
 | gen_source_file.rs:1:1:6:2 | SourceFile | 0 | gen_source_file.rs:3:1:6:1 | test_source_file |
-| lib.rs:1:1:2:21 | SourceFile | 0 | lib.rs:2:1:2:20 | Module |
+| lib.rs:1:1:1:20 | SourceFile | 0 | lib.rs:1:1:1:20 | Module |

rust/ql/test/extractor-tests/generated/options

@@ -0,0 +1 @@
+qltest_cargo_check: false

rust/ql/test/extractor-tests/utf8/ast.expected

@@ -1,6 +1,6 @@
-| lib.rs:1:1:2:22 | SourceFile |
+| lib.rs:1:1:1:21 | Module |
-| lib.rs:2:1:2:21 | Module |
+| lib.rs:1:1:1:21 | SourceFile |
-| lib.rs:2:5:2:20 | Name |
+| lib.rs:1:5:1:20 | Name |
 | utf8_identifiers.rs:1:1:4:6 | foo |
 | utf8_identifiers.rs:1:1:12:2 | SourceFile |
 | utf8_identifiers.rs:1:4:1:6 | Name |

rust/ql/test/library-tests/controlflow/options

@@ -0,0 +1 @@
+qltest_cargo_check: false

rust/ql/test/library-tests/variables/options

@@ -0,0 +1 @@
+qltest_cargo_check: true

rust/ql/test/options

@@ -0,0 +1 @@
+qltest_cargo_check: true

rust/ql/test/query-tests/options

@@ -0,0 +1 @@
+qltest_cargo_check: false

rust/tools/qltest.cmd

@@ -0,0 +1,7 @@
+@echo off
+
+set "RUST_BACKTRACE=full"
+
+type NUL && "%CODEQL_EXTRACTOR_RUST_ROOT%/tools/%CODEQL_PLATFORM%/extractor" --qltest
+
+exit /b %ERRORLEVEL%

rust/tools/qltest.sh

@@ -1,40 +1,7 @@
 #!/bin/bash
 
-mkdir -p "$CODEQL_EXTRACTOR_RUST_TRAP_DIR"
+set -eu
 
 export RUST_BACKTRACE=full
 
-QLTEST_LOG="$CODEQL_EXTRACTOR_RUST_LOG_DIR"/qltest.log
+"$CODEQL_EXTRACTOR_RUST_ROOT/tools/$CODEQL_PLATFORM/extractor" --qltest
-
-EXTRACTOR="$CODEQL_EXTRACTOR_RUST_ROOT/tools/$CODEQL_PLATFORM/extractor"
-echo > lib.rs
-for src in *.rs; do
-  if [[ "$src" == "lib.rs" ]]; then
-    continue
-  elif [[ "$src" == "main.rs" ]]; then
-    continue
-  else
-    echo "mod ${src%.rs};" >> lib.rs
-  fi
-done
-cat > Cargo.toml << EOF
-[workspace]
-[package]
-name = "test"
-version="0.0.1"
-edition="2021"
-[lib]
-path="lib.rs"
-EOF
-if [[ -f "main.rs" ]]; then
-cat >> Cargo.toml << EOF
-[[bin]]
-name = "main"
-path = "main.rs"
-EOF
-fi
-"$EXTRACTOR" *.rs >> "$QLTEST_LOG"
-if [[ "$?" != 0 ]]; then
-  cat "$QLTEST_LOG" # Show compiler errors on extraction failure
-  exit 1
-fi