Merge branch 'main' into redsun82/rust-config

rust/extractor/Cargo.toml

@@ -8,7 +8,6 @@ edition = "2021"
 anyhow = "1.0.95"
 clap = { version = "4.5.26", features = ["derive"] }
 figment = { version = "0.10.19", features = ["env", "yaml"] }
-log = "0.4.22"
 num-traits = "0.2.19"
 ra_ap_base_db = "0.0.258"
 ra_ap_hir = "0.0.258"
@@ -26,7 +25,6 @@ ra_ap_cfg = "0.0.258"
 ra_ap_intern = "0.0.258"
 serde = "1.0.217"
 serde_with = "3.12.0"
-stderrlog = "0.6.0"
 triomphe = "0.1.14"
 argfile = "0.2.1"
 codeql-extractor = { path = "../../shared/tree-sitter-extractor" }
@@ -37,3 +35,6 @@ chrono = { version = "0.4.39", features = ["serde"] }
 serde_json = "1.0.135"
 dunce = "1.0.5"
 toml = "0.8.19"
+tracing = "0.1.41"
+tracing-flame = "0.2.0"
+tracing-subscriber = "0.3.19"

rust/extractor/src/archive.rs

@@ -1,7 +1,7 @@
 use codeql_extractor::file_paths;
-use log::{debug, warn};
 use std::fs;
 use std::path::{Path, PathBuf};
+use tracing::{debug, warn};
 
 pub struct Archiver {
     pub root: PathBuf,

rust/extractor/src/config.rs

@@ -9,7 +9,6 @@ use figment::{
     Figment,
 };
 use itertools::Itertools;
-use num_traits::Zero;
 use ra_ap_cfg::{CfgAtom, CfgDiff};
 use ra_ap_ide_db::FxHashMap;
 use ra_ap_intern::Symbol;
@@ -54,7 +53,8 @@ pub struct Config {
     pub cargo_extra_env: FxHashMap<String, String>,
     pub cargo_extra_args: Vec<String>,
     pub cargo_all_targets: bool,
-    pub verbose: u8,
+    pub logging_flamegraph: Option<PathBuf>,
+    pub logging_verbosity: Option<String>,
     pub compression: Compression,
     pub inputs: Vec<PathBuf>,
     pub qltest: bool,
@@ -74,7 +74,13 @@ impl Config {
             .context("expanding parameter files")?;
         let cli_args = CliConfig::parse_from(args);
         let mut figment = Figment::new()
-            .merge(Env::raw().only(["CODEQL_VERBOSE"].as_slice()))
+            .merge(Env::raw().filter_map(|f| {
+                if f.eq("CODEQL_VERBOSITY") {
+                    Some("LOGGING_VERBOSITY".into())
+                } else {
+                    None
+                }
+            }))
             .merge(Env::prefixed("CODEQL_EXTRACTOR_RUST_"))
             .merge(Env::prefixed("CODEQL_EXTRACTOR_RUST_OPTION_"))
             .merge(Serialized::defaults(cli_args));

rust/extractor/src/diagnostics.rs

@@ -1,7 +1,6 @@
 use crate::config::Config;
 use anyhow::Context;
 use chrono::{DateTime, Utc};
-use log::{debug, info};
 use ra_ap_project_model::ProjectManifest;
 use serde::ser::SerializeMap;
 use serde::Serialize;
@@ -10,6 +9,7 @@ use std::fmt::Display;
 use std::fs::File;
 use std::path::{Path, PathBuf};
 use std::time::Instant;
+use tracing::{debug, info};
 
 #[derive(Default, Debug, Clone, Copy, Serialize)]
 #[serde(rename_all = "camelCase")]

rust/extractor/src/main.rs

@@ -3,7 +3,6 @@ use crate::rust_analyzer::path_to_file_id;
 use crate::trap::TrapId;
 use anyhow::Context;
 use archive::Archiver;
-use log::{info, warn};
 use ra_ap_hir::Semantics;
 use ra_ap_ide_db::line_index::{LineCol, LineIndex};
 use ra_ap_ide_db::RootDatabase;
@@ -17,6 +16,9 @@ use std::{
     collections::HashMap,
     path::{Path, PathBuf},
 };
+use tracing::{error, info, warn};
+use tracing_subscriber::layer::SubscriberExt;
+use tracing_subscriber::util::SubscriberInitExt;
 
 mod archive;
 mod config;
@@ -86,7 +88,7 @@ impl<'a> Extractor<'a> {
         }
         translator.emit_source_file(ast);
         translator.trap.commit().unwrap_or_else(|err| {
-            log::error!(
+            error!(
                 "Failed to write trap file for: {}: {}",
                 display_path,
                 err.to_string()
@@ -181,15 +183,27 @@ fn cwd() -> anyhow::Result<AbsPathBuf> {
 }
 
 fn main() -> anyhow::Result<()> {
-    let start = Instant::now();
     let mut cfg = config::Config::extract().context("failed to load configuration")?;
-    stderrlog::new()
-        .module(module_path!())
-        .verbosity(2 + cfg.verbose as usize)
-        .init()?;
     if cfg.qltest {
         qltest::prepare(&mut cfg)?;
     }
+    let start = Instant::now();
+    let (flame_layer, _flush_guard) = if let Some(path) = &cfg.logging_flamegraph {
+        tracing_flame::FlameLayer::with_file(path)
+            .ok()
+            .map(|(a, b)| (Some(a), Some(b)))
+            .unwrap_or((None, None))
+    } else {
+        (None, None)
+    };
+
+    tracing_subscriber::registry()
+        .with(codeql_extractor::extractor::default_subscriber_with_level(
+            "single_arch",
+            &cfg.logging_verbosity,
+        ))
+        .with(flame_layer)
+        .init();
     info!("{cfg:#?}\n");
 
     let traps = trap::TrapFileProvider::new(&cfg).context("failed to set up trap files")?;
@@ -242,6 +256,5 @@ fn main() -> anyhow::Result<()> {
             }
         }
     }
-
     extractor.emit_extraction_diagnostics(start, &cfg)
 }

rust/extractor/src/qltest.rs

@@ -2,10 +2,10 @@ use crate::config::Config;
 use anyhow::Context;
 use glob::glob;
 use itertools::Itertools;
-use log::info;
 use std::ffi::OsStr;
 use std::fs;
 use std::process::Command;
+use tracing::info;
 
 fn dump_lib() -> anyhow::Result<()> {
     let path_iterator = glob("*.rs").context("globbing test sources")?;

rust/extractor/src/rust_analyzer.rs

@@ -1,10 +1,9 @@
 use itertools::Itertools;
-use log::{debug, error, info, warn};
 use ra_ap_base_db::SourceDatabase;
 use ra_ap_hir::Semantics;
 use ra_ap_ide_db::RootDatabase;
 use ra_ap_load_cargo::{load_workspace_at, LoadCargoConfig};
-use ra_ap_paths::Utf8PathBuf;
+use ra_ap_paths::{AbsPath, Utf8PathBuf};
 use ra_ap_project_model::ProjectManifest;
 use ra_ap_project_model::{CargoConfig, ManifestPath};
 use ra_ap_span::Edition;
@@ -22,6 +21,7 @@ use std::collections::{HashMap, HashSet};
 use std::fs;
 use std::path::{Path, PathBuf};
 use std::rc::Rc;
+use tracing::{debug, error, info, trace, warn};
 use triomphe::Arc;
 
 pub enum RustAnalyzer<'a> {
@@ -52,13 +52,13 @@ impl<'a> RustAnalyzer<'a> {
         config: &CargoConfig,
         load_config: &LoadCargoConfig,
     ) -> Option<(RootDatabase, Vfs)> {
-        let progress = |t| (log::trace!("progress: {}", t));
+        let progress = |t| (trace!("progress: {}", t));
         let manifest = project.manifest_path();
 
         match load_workspace_at(manifest.as_ref(), config, load_config, &progress) {
             Ok((db, vfs, _macro_server)) => Some((db, vfs)),
             Err(err) => {
-                log::error!("failed to load workspace for {}: {}", manifest, err);
+                error!("failed to load workspace for {}: {}", manifest, err);
                 None
             }
         }
@@ -132,6 +132,8 @@ impl<'a> RustAnalyzer<'a> {
 struct CargoManifestMembersSlice {
     #[serde(default)]
     members: Vec<String>,
+    #[serde(default)]
+    exclude: Vec<String>,
 }
 
 #[derive(Deserialize)]
@@ -167,6 +169,12 @@ impl TomlReader {
     }
 }
 
+fn workspace_members_match(workspace_dir: &AbsPath, members: &[String], target: &AbsPath) -> bool {
+    members.iter().any(|p| {
+        glob::Pattern::new(workspace_dir.join(p).as_str()).is_ok_and(|p| p.matches(target.as_str()))
+    })
+}
+
 fn find_workspace(reader: &mut TomlReader, manifest: &ProjectManifest) -> Option<ProjectManifest> {
     let ProjectManifest::CargoToml(cargo) = manifest else {
         return None;
@@ -196,9 +204,12 @@ fn find_workspace(reader: &mut TomlReader, manifest: &ProjectManifest) -> Option
                 if cargo.starts_with(other.parent())
                     && reader.read(other).is_ok_and(|it| {
                         it.workspace.as_ref().is_some_and(|w| {
-                            w.members
-                                .iter()
-                                .any(|m| other.parent().join(m) == cargo.parent())
+                            workspace_members_match(other.parent(), &w.members, cargo.parent())
+                                && !workspace_members_match(
+                                    other.parent(),
+                                    &w.exclude,
+                                    cargo.parent(),
+                                )
                         })
                     }) =>
             {

rust/extractor/src/translate/base.rs

@@ -4,7 +4,6 @@ use crate::rust_analyzer::FileSemanticInformation;
 use crate::trap::{DiagnosticSeverity, TrapFile, TrapId};
 use crate::trap::{Label, TrapClass};
 use itertools::Either;
-use log::Level;
 use ra_ap_base_db::ra_salsa::InternKey;
 use ra_ap_base_db::CrateOrigin;
 use ra_ap_hir::db::ExpandDatabase;
@@ -72,6 +71,18 @@ macro_rules! emit_detached {
     ($($_:tt)*) => {};
 }
 
+// see https://github.com/tokio-rs/tracing/issues/2730
+macro_rules! dispatch_to_tracing {
+    ($lvl:ident, $($arg:tt)+) => {
+        match $lvl {
+            DiagnosticSeverity::Debug => ::tracing::debug!($($arg)+),
+            DiagnosticSeverity::Info => ::tracing::info!($($arg)+),
+            DiagnosticSeverity::Warning => ::tracing::warn!($($arg)+),
+            DiagnosticSeverity::Error => ::tracing::error!($($arg)+),
+        }
+    };
+}
+
 pub struct Translator<'a> {
     pub trap: TrapFile,
     path: &'a str,
@@ -176,20 +187,15 @@ impl<'a> Translator<'a> {
         location: (LineCol, LineCol),
     ) {
         let (start, end) = location;
-        let level = match severity {
-            DiagnosticSeverity::Debug => Level::Debug,
-            DiagnosticSeverity::Info => Level::Info,
-            DiagnosticSeverity::Warning => Level::Warn,
-            DiagnosticSeverity::Error => Level::Error,
-        };
-        log::log!(
-            level,
+        dispatch_to_tracing!(
+            severity,
             "{}:{}:{}: {}",
             self.path,
             start.line + 1,
             start.col + 1,
-            &full_message
+            &full_message,
         );
+
         if severity > DiagnosticSeverity::Debug {
             let location = self.trap.emit_location_label(self.label, start, end);
             self.trap

rust/extractor/src/trap.rs

@@ -1,12 +1,12 @@
 use crate::config::Compression;
 use crate::{config, generated};
 use codeql_extractor::{extractor, file_paths, trap};
-use log::debug;
 use ra_ap_ide_db::line_index::LineCol;
 use std::fmt::Debug;
 use std::hash::Hash;
 use std::marker::PhantomData;
 use std::path::{Path, PathBuf};
+use tracing::debug;
 
 pub use trap::Label as UntypedLabel;
 pub use trap::Writer;