Merge branch 'main' into redsun82/rust-typed-labels

rust/extractor/BUILD.bazel

@@ -7,11 +7,13 @@ codeql_rust_binary(
     aliases = aliases(),
     proc_macro_deps = all_crate_deps(
         proc_macro = True,
-    ),
+    ) + [
+        "//rust/extractor/macros",
+    ],
     visibility = ["//rust:__subpackages__"],
     deps = all_crate_deps(
         normal = True,
     ) + [
-        "//shared/tree-sitter-extractor:codeql-extractor",
+        "//shared/tree-sitter-extractor",
     ],
 )

rust/extractor/Cargo.toml

@@ -22,7 +22,6 @@ serde = "1.0.209"
 serde_with = "3.9.0"
 stderrlog = "0.6.0"
 triomphe = "0.1.13"
-# Ideally, we'd like to pull this in via a relative path.
-# However, our bazel/rust tooling chokes on this, c.f. https://github.com/bazelbuild/rules_rust/issues/1525
-# Therefore, we have a pretty bad hack in place instead, see README.md in the codeql-extractor-fake-crate directory.
+argfile = "0.2.1"
 codeql-extractor = { path = "../../shared/tree-sitter-extractor" }
+rust-extractor-macros = { path = "macros" }

rust/extractor/macros/BUILD.bazel

@@ -0,0 +1,20 @@
+load("@rules_rust//rust:defs.bzl", "rust_proc_macro")
+load("@tree_sitter_extractors_deps//:defs.bzl", "aliases", "all_crate_deps")
+
+rust_proc_macro(
+    name = "rust_extractor_macros",
+    srcs = glob(["src/**/*.rs"]),
+    aliases = aliases(),
+    proc_macro_deps = all_crate_deps(
+        proc_macro = True,
+    ),
+    deps = all_crate_deps(
+        normal = True,
+    ),
+)
+
+alias(
+    name = "macros",
+    actual = "rust_extractor_macros",
+    visibility = ["//rust:__subpackages__"],
+)

rust/extractor/macros/Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+name = "rust-extractor-macros"
+version = "0.1.0"
+edition = "2021"
+
+[lib]
+proc-macro = true
+
+[dependencies]
+quote = "1.0.37"
+syn = { version = "2.0.77", features = ["full"] }

rust/extractor/macros/src/lib.rs

@@ -0,0 +1,56 @@
+use proc_macro::TokenStream;
+use quote::{format_ident, quote};
+
+/// Allow all fields in the extractor config to be also overrideable by extractor CLI flags
+#[proc_macro_attribute]
+pub fn extractor_cli_config(_attr: TokenStream, item: TokenStream) -> TokenStream {
+    let ast = syn::parse_macro_input!(item as syn::ItemStruct);
+    let name = &ast.ident;
+    let new_name = format_ident!("Cli{}", name);
+    let fields: Vec<_> = ast
+        .fields
+        .iter()
+        .map(|f| {
+            let id = f.ident.as_ref().unwrap();
+            let ty = &f.ty;
+            if let syn::Type::Path(p) = ty {
+                if p.path.is_ident(&format_ident!("bool")) {
+                    return quote! {
+                        #[arg(long)]
+                        #[serde(skip_serializing_if="<&bool>::not")]
+                        #id: bool,
+                    };
+                }
+            }
+            if id == &format_ident!("verbose") {
+                quote! {
+                    #[arg(long, short, action=clap::ArgAction::Count)]
+                    #[serde(skip_serializing_if="u8::is_zero")]
+                    #id: u8,
+                }
+            } else if id == &format_ident!("inputs") {
+                quote! {
+                    #id: #ty,
+                }
+            } else {
+                quote! {
+                    #[arg(long)]
+                    #id: Option<#ty>,
+                }
+            }
+        })
+        .collect();
+    let gen = quote! {
+        #[serde_with::apply(_ => #[serde(default)])]
+        #[derive(Debug, Deserialize, Default)]
+        #ast
+
+        #[serde_with::skip_serializing_none]
+        #[derive(clap::Parser, Serialize)]
+        #[command(about, long_about = None)]
+        struct #new_name {
+            #(#fields)*
+        }
+    };
+    gen.into()
+}

rust/extractor/src/config.rs

@@ -1,14 +1,17 @@
 use anyhow::Context;
-use clap::{ArgAction, Parser, ValueEnum};
+use clap::Parser;
 use codeql_extractor::trap;
 use figment::{
     providers::{Env, Serialized},
     Figment,
 };
+use num_traits::Zero;
+use rust_extractor_macros::extractor_cli_config;
 use serde::{Deserialize, Serialize};
+use std::ops::Not;
 use std::path::PathBuf;
 
-#[derive(Debug, PartialEq, Eq, Default, Serialize, Deserialize, Clone, Copy, ValueEnum)]
+#[derive(Debug, PartialEq, Eq, Default, Serialize, Deserialize, Clone, Copy, clap::ValueEnum)]
 #[serde(rename_all = "lowercase")]
 #[clap(rename_all = "lowercase")]
 pub enum Compression {
@@ -26,8 +29,7 @@ impl From<Compression> for trap::Compression {
     }
 }
 
-#[serde_with::apply(_ => #[serde(default)])]
-#[derive(Debug, Deserialize, Default)]
+#[extractor_cli_config]
 pub struct Config {
     pub scratch_dir: PathBuf,
     pub trap_dir: PathBuf,
@@ -38,40 +40,13 @@ pub struct Config {
     pub inputs: Vec<PathBuf>,
 }
 
-#[serde_with::apply(_ => #[serde(skip_serializing_if = "is_default")])]
-#[derive(clap::Parser, Serialize)]
-#[command(about, long_about = None)]
-struct CliArgs {
-    #[arg(long)]
-    scratch_dir: Option<PathBuf>,
-    #[arg(long)]
-    trap_dir: Option<PathBuf>,
-    #[arg(long)]
-    source_archive_dir: Option<PathBuf>,
-    #[arg(long)]
-    compression: Option<Compression>,
-    #[arg(short, long, action = ArgAction::Count)]
-    verbose: u8,
-    #[arg(long)]
-    inputs_file: Option<PathBuf>,
-
-    inputs: Vec<PathBuf>,
-}
-
-fn is_default<T: Default + PartialEq>(t: &T) -> bool {
-    *t == Default::default()
-}
-
 impl Config {
     pub fn extract() -> anyhow::Result<Config> {
-        let mut cli_args = CliArgs::parse();
-        if let Some(inputs_file) = cli_args.inputs_file.take() {
-            let inputs_list = std::fs::read_to_string(inputs_file).context("reading file list")?;
-            cli_args
-                .inputs
-                .extend(inputs_list.split_terminator("\n").map(PathBuf::from));
-        }
+        let args = argfile::expand_args(argfile::parse_fromfile, argfile::PREFIX)
+            .context("expanding parameter files")?;
+        let cli_args = CliConfig::parse_from(args);
         Figment::new()
+            .merge(Env::prefixed("CODEQL_"))
             .merge(Env::prefixed("CODEQL_EXTRACTOR_RUST_"))
             .merge(Env::prefixed("CODEQL_EXTRACTOR_RUST_OPTION_"))
             .merge(Serialized::defaults(cli_args))

rust/extractor/src/generated/.generated.list

@@ -1,2 +1,2 @@
-mod.rs 7cdfedcd68cf8e41134daf810c1af78624082b0c3e8be6570339b1a69a5d457e 7cdfedcd68cf8e41134daf810c1af78624082b0c3e8be6570339b1a69a5d457e
-top.rs 3013fe5b706d009acbad2fe6aead56dd698cc2d41cf795b3eea49150f8fec7cd 3013fe5b706d009acbad2fe6aead56dd698cc2d41cf795b3eea49150f8fec7cd
+mod.rs 4bcb9def847469aae9d8649461546b7c21ec97cf6e63d3cf394e339915ce65d7 4bcb9def847469aae9d8649461546b7c21ec97cf6e63d3cf394e339915ce65d7
+top.rs 4bf2a559a91f46a8fadc7de0c3636d0fc9add8de5b97ae289ade79c1d02a8abd 4bf2a559a91f46a8fadc7de0c3636d0fc9add8de5b97ae289ade79c1d02a8abd

rust/extractor/src/generated/mod.rs

@@ -1,4 +1,4 @@
-// generated by codegen
+// generated by codegen, do not edit
 
 mod top;
 pub use top::*;

rust/extractor/src/generated/top.rs

@@ -1,4 +1,4 @@
-// generated by codegen
+// generated by codegen, do not edit
 
 #![cfg_attr(any(), rustfmt::skip)]