Merge pull request #16451 from hvitved/treesitter/codeql-verbosity

Tree-sitter: Respect verbosity defined in `CODEQL_VERBOSITY`

ql/extractor/src/extractor.rs

@@ -20,12 +20,7 @@ pub struct Options {
 }
 
 pub fn run(options: Options) -> std::io::Result<()> {
-    tracing_subscriber::fmt()
-        .with_target(false)
-        .without_time()
-        .with_level(true)
-        .with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
-        .init();
+    codeql_extractor::extractor::set_tracing_level("ql");
 
     let extractor = simple::Extractor {
         prefix: "ql".to_string(),

ql/extractor/src/generator.rs

@@ -15,12 +15,7 @@ pub struct Options {
 }
 
 pub fn run(options: Options) -> std::io::Result<()> {
-    tracing_subscriber::fmt()
-        .with_target(false)
-        .without_time()
-        .with_level(true)
-        .with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
-        .init();
+    codeql_extractor::extractor::set_tracing_level("ql");
 
     let languages = vec![
         Language {

ql/rust-toolchain.toml

@@ -2,6 +2,6 @@
 # extractor. It is set to the lowest version of Rust we want to support.
 
 [toolchain]
-channel = "1.70"
+channel = "1.74"
 profile = "minimal"
 components = [ "rustfmt" ]

ruby/extractor/Cargo.toml

@@ -34,7 +34,7 @@ lazy_static = "1.4.0"
 # of lock-file update time, but `rules_rust` pins generates a bazel rule that unconditionally downloads `main`, which
 # breaks build hermeticity. So, rev-pinning it is.
 # See also https://github.com/bazelbuild/rules_rust/issues/2502.
-codeql-extractor = { git = "https://github.com/github/codeql.git", rev = "bc1283c7152b0bb4d27ff6a004869f493e93d2b3" }
+codeql-extractor = { git = "https://github.com/github/codeql.git", rev = "a523be4d0a1e2420a1884f7c4f8754a7c4fb7e21" }
 
 [patch.crates-io]
 tree-sitter = {git = "https://github.com/redsun82/tree-sitter.git", rev = "1f5c1112ceaa8fc6aff61d1852690407670d2a96"}

ruby/extractor/rust-toolchain.toml

@@ -2,6 +2,6 @@
 # extractor. It is set to the lowest version of Rust we want to support.
 
 [toolchain]
-channel = "1.70"
+channel = "1.74"
 profile = "minimal"
 components = [ "rustfmt" ]

ruby/extractor/src/extractor.rs

@@ -25,15 +25,7 @@ pub struct Options {
 }
 
 pub fn run(options: Options) -> std::io::Result<()> {
-    tracing_subscriber::fmt()
-        .with_target(false)
-        .without_time()
-        .with_level(true)
-        .with_env_filter(
-            tracing_subscriber::EnvFilter::try_from_default_env()
-                .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("ruby_extractor=warn")),
-        )
-        .init();
+    extractor::set_tracing_level("ruby");
     let diagnostics = diagnostics::DiagnosticLoggers::new("ruby");
     let mut main_thread_logger = diagnostics.logger();
     let num_threads = match codeql_extractor::options::num_threads() {

ruby/extractor/src/generator.rs

@@ -15,12 +15,7 @@ pub struct Options {
 }
 
 pub fn run(options: Options) -> std::io::Result<()> {
-    tracing_subscriber::fmt()
-        .with_target(false)
-        .without_time()
-        .with_level(true)
-        .with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
-        .init();
+    codeql_extractor::extractor::set_tracing_level("ruby");
 
     let languages = vec![
         Language {

shared/tree-sitter-extractor/Cargo.toml

@@ -9,6 +9,7 @@ flate2 = "1.0"
 globset = "0.4"
 tree-sitter = ">= 0.22.6"
 tracing = "0.1"
+tracing-subscriber = { version = "0.3.3", features = ["env-filter"] }
 rayon = "1.5.0"
 regex = "1.7.1"
 encoding = "0.2"

shared/tree-sitter-extractor/rust-toolchain.toml

@@ -2,6 +2,6 @@
 # extractor. It is set to the lowest version of Rust we want to support.
 
 [toolchain]
-channel = "1.70"
+channel = "1.74"
 profile = "minimal"
 components = [ "clippy", "rustfmt" ]

shared/tree-sitter-extractor/src/extractor/mod.rs

@@ -4,12 +4,44 @@ use crate::node_types::{self, EntryKind, Field, NodeTypeMap, Storage, TypeName};
 use crate::trap;
 use std::collections::BTreeMap as Map;
 use std::collections::BTreeSet as Set;
+use std::env;
 use std::path::Path;
 
 use tree_sitter::{Language, Node, Parser, Range, Tree};
 
 pub mod simple;
 
+/// Sets the tracing level based on the environment variables
+/// `RUST_LOG` and `CODEQL_VERBOSITY` (prioritized in that order),
+/// falling back to `warn` if neither is set.
+pub fn set_tracing_level(language: &str) {
+    tracing_subscriber::fmt()
+        .with_target(false)
+        .without_time()
+        .with_level(true)
+        .with_env_filter(
+            tracing_subscriber::EnvFilter::try_from_default_env().unwrap_or_else(
+                |_| -> tracing_subscriber::EnvFilter {
+                    let verbosity = env::var("CODEQL_VERBOSITY")
+                        .map(|v| match v.to_lowercase().as_str() {
+                            "off" | "errors" => "error",
+                            "warnings" => "warn",
+                            "info" | "progress" => "info",
+                            "debug" | "progress+" => "debug",
+                            "trace" | "progress++" | "progress+++" => "trace",
+                            _ => "warn",
+                        })
+                        .unwrap_or_else(|_| "warn");
+                    tracing_subscriber::EnvFilter::new(format!(
+                        "{}_extractor={}",
+                        language, verbosity
+                    ))
+                },
+            ),
+        )
+        .init();
+}
+
 pub fn populate_file(writer: &mut trap::Writer, absolute_path: &Path) -> trap::Label {
     let (file_label, fresh) = writer.global_id(&trap::full_id_for_file(
         &file_paths::normalize_path(absolute_path),