From 1b3a5cdab138855838b41c75998e59b7d8ee191d Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Wed, 11 Sep 2024 17:09:18 +0200 Subject: [PATCH 01/57] Rust: make the cli flags override automatic This makes the clap flags overlay over `Config` entirely derived via an attribute macro. Also, the `--intputs-file` option is replaced by a more standard and versatile `@` parameter file mechanism. --- Cargo.lock | 38 +++++++++++++++++++++++ Cargo.toml | 1 + rust/extractor/Cargo.toml | 5 ++- rust/extractor/macros/Cargo.toml | 11 +++++++ rust/extractor/macros/src/lib.rs | 52 ++++++++++++++++++++++++++++++++ rust/extractor/src/config.rs | 41 ++++--------------------- rust/tools/index-files.sh | 2 +- 7 files changed, 111 insertions(+), 39 deletions(-) create mode 100644 rust/extractor/macros/Cargo.toml create mode 100644 rust/extractor/macros/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index fcd7d806843..d3b54343499 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -96,6 +96,16 @@ version = "1.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "10f00e1f6e58a40e807377c75c6a7f97bf9044fab57816f2414e6f5f4499d7b8" +[[package]] +name = "argfile" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a1cc0ba69de57db40674c66f7cf2caee3981ddef084388482c95c0e2133e5e8" +dependencies = [ + "fs-err", + "os_str_bytes", +] + [[package]] name = "arrayvec" version = "0.7.6" @@ -360,6 +370,7 @@ name = "codeql-rust" version = "0.1.0" dependencies = [ "anyhow", + "argfile", "clap", "codeql-extractor", "figment", @@ -374,6 +385,7 @@ dependencies = [ "ra_ap_project_model", "ra_ap_syntax", "ra_ap_vfs", + "rust-extractor-macros", "serde", "serde_with", "stderrlog", @@ -643,6 +655,15 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "fs-err" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88a41f105fe1d5b6b34b2055e3dc59bb79b46b48b2040b9e6c7b4b5de097aa41" +dependencies = [ + "autocfg", +] + [[package]] name = "fsevent-sys" version = "4.1.0" @@ -1064,6 +1085,15 @@ version = "11.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b410bbe7e14ab526a0e86877eb47c6996a2bd7746f027ba551028c925390e4e9" +[[package]] +name = "os_str_bytes" +version = "7.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ac44c994af577c799b1b4bd80dc214701e349873ad894d6cdf96f4f7526e0b9" +dependencies = [ + "memchr", +] + [[package]] name = "overload" version = "0.1.1" @@ -1875,6 +1905,14 @@ dependencies = [ "text-size", ] +[[package]] +name = "rust-extractor-macros" +version = "0.1.0" +dependencies = [ + "quote", + "syn", +] + [[package]] name = "rustc-hash" version = "1.1.0" diff --git a/Cargo.toml b/Cargo.toml index 5f095736c8a..4aacef79adc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,7 @@ members = [ "shared/tree-sitter-extractor", "ruby/extractor", "rust/extractor", + "rust/extractor/macros", ] [patch.crates-io] diff --git a/rust/extractor/Cargo.toml b/rust/extractor/Cargo.toml index c849ea4aa46..3b474f90f98 100644 --- a/rust/extractor/Cargo.toml +++ b/rust/extractor/Cargo.toml @@ -22,7 +22,6 @@ serde = "1.0.209" serde_with = "3.9.0" stderrlog = "0.6.0" triomphe = "0.1.13" -# Ideally, we'd like to pull this in via a relative path. -# However, our bazel/rust tooling chokes on this, c.f. https://github.com/bazelbuild/rules_rust/issues/1525 -# Therefore, we have a pretty bad hack in place instead, see README.md in the codeql-extractor-fake-crate directory. +argfile = "0.2.1" codeql-extractor = { path = "../../shared/tree-sitter-extractor" } +rust-extractor-macros = { path = "macros" } diff --git a/rust/extractor/macros/Cargo.toml b/rust/extractor/macros/Cargo.toml new file mode 100644 index 00000000000..d4d10bc3bde --- /dev/null +++ b/rust/extractor/macros/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "rust-extractor-macros" +version = "0.1.0" +edition = "2021" + +[lib] +proc-macro = true + +[dependencies] +quote = "1.0.37" +syn = { version = "2.0.77", features = ["full"] } diff --git a/rust/extractor/macros/src/lib.rs b/rust/extractor/macros/src/lib.rs new file mode 100644 index 00000000000..13472665454 --- /dev/null +++ b/rust/extractor/macros/src/lib.rs @@ -0,0 +1,52 @@ +use proc_macro::TokenStream; +use quote::{quote, format_ident}; +use syn; + + +/// Allow all fields in the extractor config to be also overrideable by extractor CLI flags +#[proc_macro_attribute] +pub fn extractor_cli_config(_attr: TokenStream, item: TokenStream) -> TokenStream { + let ast = syn::parse_macro_input!(item as syn::ItemStruct); + let name = &ast.ident; + let new_name = format_ident!("Cli{}", name); + let fields: Vec<_> = ast.fields.iter().map(|f| { + let id = f.ident.as_ref().unwrap(); + let ty = &f.ty; + if let syn::Type::Path(p) = ty { + if p.path.is_ident(&format_ident!("bool")) { + return quote! { + #[arg(long)] + #id: bool, + }; + } + } + if id == &format_ident!("verbose") { + quote! { + #[arg(long, short, action=clap::ArgAction::Count)] + #id: u8, + } + } else if id == &format_ident!("inputs") { + quote! { + #id: #ty, + } + } else { + quote! { + #[arg(long)] + #id: Option<#ty>, + } + } + }).collect(); + let gen = quote! { + #[serde_with::apply(_ => #[serde(default)])] + #[derive(Debug, Deserialize, Default)] + #ast + + #[serde_with::skip_serializing_none] + #[derive(clap::Parser, Serialize)] + #[command(about, long_about = None)] + struct #new_name { + #(#fields)* + } + }; + gen.into() +} diff --git a/rust/extractor/src/config.rs b/rust/extractor/src/config.rs index 399c2bb9e7e..310ca2c3649 100644 --- a/rust/extractor/src/config.rs +++ b/rust/extractor/src/config.rs @@ -1,14 +1,15 @@ use anyhow::Context; -use clap::{ArgAction, Parser, ValueEnum}; +use clap::Parser; use codeql_extractor::trap; use figment::{ providers::{Env, Serialized}, Figment, }; +use rust_extractor_macros::extractor_cli_config; use serde::{Deserialize, Serialize}; use std::path::PathBuf; -#[derive(Debug, PartialEq, Eq, Default, Serialize, Deserialize, Clone, Copy, ValueEnum)] +#[derive(Debug, PartialEq, Eq, Default, Serialize, Deserialize, Clone, Copy, clap::ValueEnum)] #[serde(rename_all = "lowercase")] #[clap(rename_all = "lowercase")] pub enum Compression { @@ -26,8 +27,7 @@ impl From for trap::Compression { } } -#[serde_with::apply(_ => #[serde(default)])] -#[derive(Debug, Deserialize, Default)] +#[extractor_cli_config] pub struct Config { pub scratch_dir: PathBuf, pub trap_dir: PathBuf, @@ -38,39 +38,10 @@ pub struct Config { pub inputs: Vec, } -#[serde_with::apply(_ => #[serde(skip_serializing_if = "is_default")])] -#[derive(clap::Parser, Serialize)] -#[command(about, long_about = None)] -struct CliArgs { - #[arg(long)] - scratch_dir: Option, - #[arg(long)] - trap_dir: Option, - #[arg(long)] - source_archive_dir: Option, - #[arg(long)] - compression: Option, - #[arg(short, long, action = ArgAction::Count)] - verbose: u8, - #[arg(long)] - inputs_file: Option, - - inputs: Vec, -} - -fn is_default(t: &T) -> bool { - *t == Default::default() -} - impl Config { pub fn extract() -> anyhow::Result { - let mut cli_args = CliArgs::parse(); - if let Some(inputs_file) = cli_args.inputs_file.take() { - let inputs_list = std::fs::read_to_string(inputs_file).context("reading file list")?; - cli_args - .inputs - .extend(inputs_list.split_terminator("\n").map(PathBuf::from)); - } + let args = argfile::expand_args(argfile::parse_fromfile, argfile::PREFIX)?; + let cli_args = CliConfig::parse_from(args); Figment::new() .merge(Env::prefixed("CODEQL_EXTRACTOR_RUST_")) .merge(Env::prefixed("CODEQL_EXTRACTOR_RUST_OPTION_")) diff --git a/rust/tools/index-files.sh b/rust/tools/index-files.sh index da4b841b692..f3d93fbaf4a 100755 --- a/rust/tools/index-files.sh +++ b/rust/tools/index-files.sh @@ -2,4 +2,4 @@ set -eu -exec "$CODEQL_EXTRACTOR_RUST_ROOT/tools/$CODEQL_PLATFORM/extractor" --inputs-file="$1" +exec "$CODEQL_EXTRACTOR_RUST_ROOT/tools/$CODEQL_PLATFORM/extractor" @"$1" From 0a8c0f5ab40cfc681913ce20ae1328c195ef53fa Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Thu, 12 Sep 2024 08:46:50 +0200 Subject: [PATCH 02/57] Rust: fix bazel build --- MODULE.bazel | 1 + ruby/extractor/BUILD.bazel | 2 +- rust/extractor/BUILD.bazel | 6 ++++-- rust/extractor/macros/BUILD.bazel | 20 ++++++++++++++++++++ shared/tree-sitter-extractor/BUILD.bazel | 8 ++++++-- 5 files changed, 32 insertions(+), 5 deletions(-) create mode 100644 rust/extractor/macros/BUILD.bazel diff --git a/MODULE.bazel b/MODULE.bazel index 20efc539039..020d5b0fbae 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -60,6 +60,7 @@ r.from_cargo( "//:Cargo.toml", "//ruby/extractor:Cargo.toml", "//rust/extractor:Cargo.toml", + "//rust/extractor/macros:Cargo.toml", "//shared/tree-sitter-extractor:Cargo.toml", ], ) diff --git a/ruby/extractor/BUILD.bazel b/ruby/extractor/BUILD.bazel index 203f90310fa..158f1c91e4c 100644 --- a/ruby/extractor/BUILD.bazel +++ b/ruby/extractor/BUILD.bazel @@ -12,6 +12,6 @@ codeql_rust_binary( deps = all_crate_deps( normal = True, ) + [ - "//shared/tree-sitter-extractor:codeql-extractor", + "//shared/tree-sitter-extractor", ], ) diff --git a/rust/extractor/BUILD.bazel b/rust/extractor/BUILD.bazel index 924d5e01497..a5d99e82584 100644 --- a/rust/extractor/BUILD.bazel +++ b/rust/extractor/BUILD.bazel @@ -7,11 +7,13 @@ codeql_rust_binary( aliases = aliases(), proc_macro_deps = all_crate_deps( proc_macro = True, - ), + ) + [ + "//rust/extractor/macros", + ], visibility = ["//rust:__subpackages__"], deps = all_crate_deps( normal = True, ) + [ - "//shared/tree-sitter-extractor:codeql-extractor", + "//shared/tree-sitter-extractor", ], ) diff --git a/rust/extractor/macros/BUILD.bazel b/rust/extractor/macros/BUILD.bazel new file mode 100644 index 00000000000..4ddfb14a171 --- /dev/null +++ b/rust/extractor/macros/BUILD.bazel @@ -0,0 +1,20 @@ +load("@rules_rust//rust:defs.bzl", "rust_proc_macro") +load("@tree_sitter_extractors_deps//:defs.bzl", "aliases", "all_crate_deps") + +rust_proc_macro( + name = "rust_extractor_macros", + srcs = glob(["src/**/*.rs"]), + aliases = aliases(), + proc_macro_deps = all_crate_deps( + proc_macro = True, + ), + deps = all_crate_deps( + normal = True, + ), +) + +alias( + name = "macros", + actual = "rust_extractor_macros", + visibility = ["//rust:__subpackages__"], +) diff --git a/shared/tree-sitter-extractor/BUILD.bazel b/shared/tree-sitter-extractor/BUILD.bazel index dc9001a32d2..0ebc189954b 100644 --- a/shared/tree-sitter-extractor/BUILD.bazel +++ b/shared/tree-sitter-extractor/BUILD.bazel @@ -1,8 +1,6 @@ load("@rules_rust//rust:defs.bzl", "rust_library") load("@tree_sitter_extractors_deps//:defs.bzl", "aliases", "all_crate_deps") -package(default_visibility = ["//visibility:public"]) - rust_library( name = "codeql-extractor", srcs = glob([ @@ -14,3 +12,9 @@ rust_library( ], deps = all_crate_deps(), ) + +alias( + name = "tree-sitter-extractor", + actual = ":codeql-extractor", + visibility = ["//visibility:public"], +) From 6adf88542e4477f7cab6ff02ca965b293f85ce50 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Thu, 12 Sep 2024 08:53:08 +0200 Subject: [PATCH 03/57] Rust: fix linting script --- rust/extractor/macros/src/lib.rs | 58 +++++++++++++++++--------------- rust/lint.py | 10 +++--- 2 files changed, 36 insertions(+), 32 deletions(-) diff --git a/rust/extractor/macros/src/lib.rs b/rust/extractor/macros/src/lib.rs index 13472665454..1905d40f9b2 100644 --- a/rust/extractor/macros/src/lib.rs +++ b/rust/extractor/macros/src/lib.rs @@ -1,7 +1,5 @@ use proc_macro::TokenStream; -use quote::{quote, format_ident}; -use syn; - +use quote::{format_ident, quote}; /// Allow all fields in the extractor config to be also overrideable by extractor CLI flags #[proc_macro_attribute] @@ -9,33 +7,37 @@ pub fn extractor_cli_config(_attr: TokenStream, item: TokenStream) -> TokenStrea let ast = syn::parse_macro_input!(item as syn::ItemStruct); let name = &ast.ident; let new_name = format_ident!("Cli{}", name); - let fields: Vec<_> = ast.fields.iter().map(|f| { - let id = f.ident.as_ref().unwrap(); - let ty = &f.ty; - if let syn::Type::Path(p) = ty { - if p.path.is_ident(&format_ident!("bool")) { - return quote! { + let fields: Vec<_> = ast + .fields + .iter() + .map(|f| { + let id = f.ident.as_ref().unwrap(); + let ty = &f.ty; + if let syn::Type::Path(p) = ty { + if p.path.is_ident(&format_ident!("bool")) { + return quote! { + #[arg(long)] + #id: bool, + }; + } + } + if id == &format_ident!("verbose") { + quote! { + #[arg(long, short, action=clap::ArgAction::Count)] + #id: u8, + } + } else if id == &format_ident!("inputs") { + quote! { + #id: #ty, + } + } else { + quote! { #[arg(long)] - #id: bool, - }; + #id: Option<#ty>, + } } - } - if id == &format_ident!("verbose") { - quote! { - #[arg(long, short, action=clap::ArgAction::Count)] - #id: u8, - } - } else if id == &format_ident!("inputs") { - quote! { - #id: #ty, - } - } else { - quote! { - #[arg(long)] - #id: Option<#ty>, - } - } - }).collect(); + }) + .collect(); let gen = quote! { #[serde_with::apply(_ => #[serde(default)])] #[derive(Debug, Deserialize, Default)] diff --git a/rust/lint.py b/rust/lint.py index 1af2470dbbc..eb71cbd0b3d 100755 --- a/rust/lint.py +++ b/rust/lint.py @@ -5,12 +5,14 @@ import pathlib import shutil import sys -extractor_dir = pathlib.Path(__file__).resolve().parent / "extractor" +this_dir = pathlib.Path(__file__).resolve().parent cargo = shutil.which("cargo") assert cargo, "no cargo binary found on `PATH`" -fmt = subprocess.run([cargo, "fmt", "--quiet"], cwd=extractor_dir) -clippy = subprocess.run([cargo, "clippy", "--fix", "--allow-dirty", "--allow-staged", "--quiet"], - cwd=extractor_dir) +fmt = subprocess.run([cargo, "fmt", "--all", "--quiet"], cwd=this_dir) +for manifest in this_dir.rglob("Cargo.toml"): + if not manifest.is_relative_to(this_dir / "ql"): + clippy = subprocess.run([cargo, "clippy", "--fix", "--allow-dirty", "--allow-staged", "--quiet"], + cwd=manifest.parent) sys.exit(fmt.returncode or clippy.returncode) From 5ae88243033c013bf1fd473e1e9e0648493e7aed Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Thu, 12 Sep 2024 08:56:07 +0200 Subject: [PATCH 04/57] Rust: add context to parameter file expansion errors --- rust/extractor/src/config.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rust/extractor/src/config.rs b/rust/extractor/src/config.rs index 310ca2c3649..74c9f4b20a5 100644 --- a/rust/extractor/src/config.rs +++ b/rust/extractor/src/config.rs @@ -40,7 +40,8 @@ pub struct Config { impl Config { pub fn extract() -> anyhow::Result { - let args = argfile::expand_args(argfile::parse_fromfile, argfile::PREFIX)?; + let args = argfile::expand_args(argfile::parse_fromfile, argfile::PREFIX) + .context("expanding parameter files")?; let cli_args = CliConfig::parse_from(args); Figment::new() .merge(Env::prefixed("CODEQL_EXTRACTOR_RUST_")) From 403cc3df9013c3a278b6b8b9b8b5a52f2dd2d91b Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 13 Sep 2024 06:50:12 +0200 Subject: [PATCH 05/57] Rust: avoid cli flag defaults overriding env settings --- rust/extractor/macros/src/lib.rs | 2 ++ rust/extractor/src/config.rs | 2 ++ 2 files changed, 4 insertions(+) diff --git a/rust/extractor/macros/src/lib.rs b/rust/extractor/macros/src/lib.rs index 1905d40f9b2..781d53bd851 100644 --- a/rust/extractor/macros/src/lib.rs +++ b/rust/extractor/macros/src/lib.rs @@ -17,6 +17,7 @@ pub fn extractor_cli_config(_attr: TokenStream, item: TokenStream) -> TokenStrea if p.path.is_ident(&format_ident!("bool")) { return quote! { #[arg(long)] + #[serde(skip_serializing_if="<&bool>::not")] #id: bool, }; } @@ -24,6 +25,7 @@ pub fn extractor_cli_config(_attr: TokenStream, item: TokenStream) -> TokenStrea if id == &format_ident!("verbose") { quote! { #[arg(long, short, action=clap::ArgAction::Count)] + #[serde(skip_serializing_if="u8::is_zero")] #id: u8, } } else if id == &format_ident!("inputs") { diff --git a/rust/extractor/src/config.rs b/rust/extractor/src/config.rs index 74c9f4b20a5..98dac7fce32 100644 --- a/rust/extractor/src/config.rs +++ b/rust/extractor/src/config.rs @@ -5,8 +5,10 @@ use figment::{ providers::{Env, Serialized}, Figment, }; +use num_traits::Zero; use rust_extractor_macros::extractor_cli_config; use serde::{Deserialize, Serialize}; +use std::ops::Not; use std::path::PathBuf; #[derive(Debug, PartialEq, Eq, Default, Serialize, Deserialize, Clone, Copy, clap::ValueEnum)] From 194c2fa9c48907ccf0ca75ecd3c3007d114d6ce7 Mon Sep 17 00:00:00 2001 From: Florin Coada Date: Fri, 13 Sep 2024 10:18:04 +0100 Subject: [PATCH 06/57] Add changedocs for 2.18.4 --- .../codeql-changelog/codeql-cli-2.18.4.rst | 45 +++++++++++++++++++ .../codeql-changelog/index.rst | 1 + 2 files changed, 46 insertions(+) create mode 100644 docs/codeql/codeql-overview/codeql-changelog/codeql-cli-2.18.4.rst diff --git a/docs/codeql/codeql-overview/codeql-changelog/codeql-cli-2.18.4.rst b/docs/codeql/codeql-overview/codeql-changelog/codeql-cli-2.18.4.rst new file mode 100644 index 00000000000..14f7cb96647 --- /dev/null +++ b/docs/codeql/codeql-overview/codeql-changelog/codeql-cli-2.18.4.rst @@ -0,0 +1,45 @@ +.. _codeql-cli-2.18.4: + +========================== +CodeQL 2.18.4 (2024-09-12) +========================== + +.. contents:: Contents + :depth: 2 + :local: + :backlinks: none + +This is an overview of changes in the CodeQL CLI and relevant CodeQL query and library packs. For additional updates on changes to the CodeQL code scanning experience, check out the `code scanning section on the GitHub blog `__, `relevant GitHub Changelog updates `__, `changes in the CodeQL extension for Visual Studio Code `__, and the `CodeQL Action changelog `__. + +Security Coverage +----------------- + +CodeQL 2.18.4 runs a total of 425 security queries when configured with the Default suite (covering 164 CWE). The Extended suite enables an additional 128 queries (covering 34 more CWE). + +CodeQL CLI +---------- + +New Features +~~~~~~~~~~~~ + +* C# support for :code:`build-mode: none` is now out of beta, and generally available. +* Go 1.23 is now supported. + +Language Libraries +------------------ + +Major Analysis Improvements +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Golang +"""""" + +* Go 1.23 is now supported. + +New Features +~~~~~~~~~~~~ + +C# +"" + +* C# support for :code:`build-mode: none` is now out of beta, and generally available. diff --git a/docs/codeql/codeql-overview/codeql-changelog/index.rst b/docs/codeql/codeql-overview/codeql-changelog/index.rst index d8baa12e995..67722290b36 100644 --- a/docs/codeql/codeql-overview/codeql-changelog/index.rst +++ b/docs/codeql/codeql-overview/codeql-changelog/index.rst @@ -11,6 +11,7 @@ A list of queries for each suite and language `is available here Date: Fri, 13 Sep 2024 13:39:39 +0200 Subject: [PATCH 07/57] Rust: add `CODEQL_` base env layer --- rust/extractor/src/config.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/rust/extractor/src/config.rs b/rust/extractor/src/config.rs index 98dac7fce32..0625968a4af 100644 --- a/rust/extractor/src/config.rs +++ b/rust/extractor/src/config.rs @@ -46,6 +46,7 @@ impl Config { .context("expanding parameter files")?; let cli_args = CliConfig::parse_from(args); Figment::new() + .merge(Env::prefixed("CODEQL_")) .merge(Env::prefixed("CODEQL_EXTRACTOR_RUST_")) .merge(Env::prefixed("CODEQL_EXTRACTOR_RUST_OPTION_")) .merge(Serialized::defaults(cli_args)) From faf1eeeb0db2c8d38be608283e1c550fad258bdc Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 13 Sep 2024 13:18:33 +0200 Subject: [PATCH 08/57] Rust: introduce typed labels --- misc/codegen/generators/rustgen.py | 16 +- misc/codegen/lib/rust.py | 3 +- misc/codegen/templates/rust_classes.mustache | 75 +- rust/extractor/src/generated/.generated.list | 2 +- rust/extractor/src/generated/top.rs | 4820 +++++++++++++++-- rust/extractor/src/translate.rs | 723 +-- rust/extractor/src/trap.rs | 65 +- rust/ql/.generated.list | 13 +- rust/ql/.gitattributes | 1 - .../rust/elements/TypeRefConstructor.qll | 14 - .../lib/codeql/rust/generated/ParentChild.qll | 41 +- rust/ql/lib/codeql/rust/generated/Raw.qll | 18 +- rust/ql/lib/codeql/rust/generated/Synth.qll | 49 +- .../rust/generated/SynthConstructors.qll | 1 - rust/ql/lib/codeql/rust/generated/TypeRef.qll | 4 +- .../codeql/rust/generated/Unimplemented.qll | 5 +- rust/ql/lib/rust.dbscheme | 16 +- rust/schema.py | 22 +- 18 files changed, 4922 insertions(+), 966 deletions(-) delete mode 100644 rust/ql/lib/codeql/rust/elements/TypeRefConstructor.qll diff --git a/misc/codegen/generators/rustgen.py b/misc/codegen/generators/rustgen.py index 9b850c3cf1b..6fa5d4a05ec 100644 --- a/misc/codegen/generators/rustgen.py +++ b/misc/codegen/generators/rustgen.py @@ -20,7 +20,7 @@ def _get_type(t: str) -> str: case "int": return "usize" case _ if t[0].isupper(): - return "trap::Label" + return f"{t}TrapLabel" case "boolean": assert False, "boolean unsupported" case _: @@ -57,6 +57,15 @@ def _get_properties( yield cls, p +def _get_ancestors( + cls: schema.Class, lookup: dict[str, schema.Class] +) -> typing.Iterable[schema.Class]: + for b in cls.bases: + base = lookup[b] + yield base + yield from _get_ancestors(base, lookup) + + class Processor: def __init__(self, data: schema.Schema): self._classmap = data.classes @@ -69,14 +78,15 @@ class Processor: _get_field(c, p) for c, p in _get_properties(cls, self._classmap) if "rust_skip" not in p.pragmas and not p.synth - ], + ] if not cls.derived else [], + ancestors=sorted(set(a.name for a in _get_ancestors(cls, self._classmap))), table_name=inflection.tableize(cls.name), ) def get_classes(self): ret = {"": []} for k, cls in self._classmap.items(): - if not cls.synth and not cls.derived: + if not cls.synth: ret.setdefault(cls.group, []).append(self._get_class(cls.name)) return ret diff --git a/misc/codegen/lib/rust.py b/misc/codegen/lib/rust.py index ac7bf4313d3..0f4b410db70 100644 --- a/misc/codegen/lib/rust.py +++ b/misc/codegen/lib/rust.py @@ -110,8 +110,9 @@ class Field: @dataclasses.dataclass class Class: name: str - table_name: str + table_name: str | None = None fields: list[Field] = dataclasses.field(default_factory=list) + ancestors: list[str] = dataclasses.field(default_factory=list) @property def single_field_entries(self): diff --git a/misc/codegen/templates/rust_classes.mustache b/misc/codegen/templates/rust_classes.mustache index 3b415683d5f..f749733b2bd 100644 --- a/misc/codegen/templates/rust_classes.mustache +++ b/misc/codegen/templates/rust_classes.mustache @@ -2,48 +2,77 @@ #![cfg_attr(any(), rustfmt::skip)] -use crate::trap::{TrapId, TrapEntry}; -use codeql_extractor::trap; +use crate::trap; {{#classes}} +#[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub struct {{name}}TrapLabel(trap::UntypedLabel); + +impl From for {{name}}TrapLabel { + fn from(value: trap::UntypedLabel) -> Self { + Self(value) + } +} + +impl From<{{name}}TrapLabel> for trap::TrapId<{{name}}> { + fn from(value: {{name}}TrapLabel) -> Self { + Self::Label(value) + } +} + +impl trap::Label for {{name}}TrapLabel { + fn as_untyped(&self) -> trap::UntypedLabel { + self.0 + } +} + +impl From<{{name}}TrapLabel> for trap::Arg { + fn from(value: {{name}}TrapLabel) -> Self { + value.0.into() + } +} + +{{#table_name}} #[derive(Debug)] pub struct {{name}} { - pub id: TrapId, + pub id: trap::TrapId<{{name}}>, {{#fields}} pub {{field_name}}: {{type}}, {{/fields}} } -impl TrapEntry for {{name}} { - fn extract_id(&mut self) -> TrapId { - std::mem::replace(&mut self.id, TrapId::Star) +impl trap::TrapEntry for {{name}} { + fn class_name() -> &'static str { "{{name}}" } + + fn extract_id(&mut self) -> trap::TrapId { + std::mem::replace(&mut self.id, trap::TrapId::Star) } - fn emit(self, id: trap::Label, out: &mut trap::Writer) { + fn emit(self, id: Self::Label, out: &mut trap::Writer) { {{#single_field_entries}} - out.add_tuple("{{table_name}}", vec![trap::Arg::Label(id){{#fields}}, self.{{field_name}}.into(){{/fields}}]); + out.add_tuple("{{table_name}}", vec![id.into(){{#fields}}, self.{{field_name}}.into(){{/fields}}]); {{/single_field_entries}} {{#fields}} {{#is_predicate}} if self.{{field_name}} { - out.add_tuple("{{table_name}}", vec![trap::Arg::Label(id)]); + out.add_tuple("{{table_name}}", vec![id.into()]); } {{/is_predicate}} {{#is_optional}} {{^is_repeated}} if let Some(v) = self.{{field_name}} { - out.add_tuple("{{table_name}}", vec![trap::Arg::Label(id), v.into()]); + out.add_tuple("{{table_name}}", vec![id.into(), v.into()]); } {{/is_repeated}} {{/is_optional}} {{#is_repeated}} for (i, v) in self.{{field_name}}.into_iter().enumerate() { {{^is_optional}} - out.add_tuple("{{table_name}}", vec![trap::Arg::Label(id){{^is_unordered}}, i.into(){{/is_unordered}}, v.into()]); + out.add_tuple("{{table_name}}", vec![id.into(){{^is_unordered}}, i.into(){{/is_unordered}}, v.into()]); {{/is_optional}} {{#is_optional}} if let Some(v) = v { - out.add_tuple("{{table_name}}", vec![trap::Arg::Label(id){{^is_unordered}}, i.into(){{/is_unordered}}, v.into()]); + out.add_tuple("{{table_name}}", vec![id.into(){{^is_unordered}}, i.into(){{/is_unordered}}, v.into()]); } {{/is_optional}} } @@ -51,4 +80,26 @@ impl TrapEntry for {{name}} { {{/fields}} } } +{{/table_name}} +{{^table_name}} +{{! virtual class, make it unbuildable }} +pub struct {{name}} { + unused: () +} +{{/table_name}} + +impl trap::TrapClass for {{name}} { + type Label = {{name}}TrapLabel; +} +{{/classes}} + +// Conversions +{{#classes}} +{{#ancestors}} +impl From<{{name}}TrapLabel> for {{.}}TrapLabel { + fn from(value: {{name}}TrapLabel) -> Self { + value.0.into() + } +} +{{/ancestors}} {{/classes}} diff --git a/rust/extractor/src/generated/.generated.list b/rust/extractor/src/generated/.generated.list index b404b2e7541..fd7cc5c6614 100644 --- a/rust/extractor/src/generated/.generated.list +++ b/rust/extractor/src/generated/.generated.list @@ -1,2 +1,2 @@ mod.rs 7cdfedcd68cf8e41134daf810c1af78624082b0c3e8be6570339b1a69a5d457e 7cdfedcd68cf8e41134daf810c1af78624082b0c3e8be6570339b1a69a5d457e -top.rs 7150acaeab0b57039ca9f2ed20311229aab5fd48b533f13410ecc34fd8e3bda0 7150acaeab0b57039ca9f2ed20311229aab5fd48b533f13410ecc34fd8e3bda0 +top.rs e06dc90de4abd57719786fd5e49e6ea3089ec3ec167c64446e25d95a16b1714c e06dc90de4abd57719786fd5e49e6ea3089ec3ec167c64446e25d95a16b1714c diff --git a/rust/extractor/src/generated/top.rs b/rust/extractor/src/generated/top.rs index de6d2106c1d..084fdbb61f0 100644 --- a/rust/extractor/src/generated/top.rs +++ b/rust/extractor/src/generated/top.rs @@ -2,1222 +2,5008 @@ #![cfg_attr(any(), rustfmt::skip)] -use crate::trap::{TrapId, TrapEntry}; -use codeql_extractor::trap; +use crate::trap; + +#[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub struct ElementTrapLabel(trap::UntypedLabel); + +impl From for ElementTrapLabel { + fn from(value: trap::UntypedLabel) -> Self { + Self(value) + } +} + +impl From for trap::TrapId { + fn from(value: ElementTrapLabel) -> Self { + Self::Label(value) + } +} + +impl trap::Label for ElementTrapLabel { + fn as_untyped(&self) -> trap::UntypedLabel { + self.0 + } +} + +impl From for trap::Arg { + fn from(value: ElementTrapLabel) -> Self { + value.0.into() + } +} + +#[derive(Debug)] +pub struct Element { + pub id: trap::TrapId, +} + +impl trap::TrapEntry for Element { + fn class_name() -> &'static str { "Element" } + + fn extract_id(&mut self) -> trap::TrapId { + std::mem::replace(&mut self.id, trap::TrapId::Star) + } + + fn emit(self, id: Self::Label, out: &mut trap::Writer) { + out.add_tuple("elements", vec![id.into()]); + } +} + +impl trap::TrapClass for Element { + type Label = ElementTrapLabel; +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub struct LocatableTrapLabel(trap::UntypedLabel); + +impl From for LocatableTrapLabel { + fn from(value: trap::UntypedLabel) -> Self { + Self(value) + } +} + +impl From for trap::TrapId { + fn from(value: LocatableTrapLabel) -> Self { + Self::Label(value) + } +} + +impl trap::Label for LocatableTrapLabel { + fn as_untyped(&self) -> trap::UntypedLabel { + self.0 + } +} + +impl From for trap::Arg { + fn from(value: LocatableTrapLabel) -> Self { + value.0.into() + } +} + +#[derive(Debug)] +pub struct Locatable { + pub id: trap::TrapId, +} + +impl trap::TrapEntry for Locatable { + fn class_name() -> &'static str { "Locatable" } + + fn extract_id(&mut self) -> trap::TrapId { + std::mem::replace(&mut self.id, trap::TrapId::Star) + } + + fn emit(self, id: Self::Label, out: &mut trap::Writer) { + out.add_tuple("locatables", vec![id.into()]); + } +} + +impl trap::TrapClass for Locatable { + type Label = LocatableTrapLabel; +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub struct AstNodeTrapLabel(trap::UntypedLabel); + +impl From for AstNodeTrapLabel { + fn from(value: trap::UntypedLabel) -> Self { + Self(value) + } +} + +impl From for trap::TrapId { + fn from(value: AstNodeTrapLabel) -> Self { + Self::Label(value) + } +} + +impl trap::Label for AstNodeTrapLabel { + fn as_untyped(&self) -> trap::UntypedLabel { + self.0 + } +} + +impl From for trap::Arg { + fn from(value: AstNodeTrapLabel) -> Self { + value.0.into() + } +} + +#[derive(Debug)] +pub struct AstNode { + pub id: trap::TrapId, +} + +impl trap::TrapEntry for AstNode { + fn class_name() -> &'static str { "AstNode" } + + fn extract_id(&mut self) -> trap::TrapId { + std::mem::replace(&mut self.id, trap::TrapId::Star) + } + + fn emit(self, id: Self::Label, out: &mut trap::Writer) { + out.add_tuple("ast_nodes", vec![id.into()]); + } +} + +impl trap::TrapClass for AstNode { + type Label = AstNodeTrapLabel; +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub struct DeclarationTrapLabel(trap::UntypedLabel); + +impl From for DeclarationTrapLabel { + fn from(value: trap::UntypedLabel) -> Self { + Self(value) + } +} + +impl From for trap::TrapId { + fn from(value: DeclarationTrapLabel) -> Self { + Self::Label(value) + } +} + +impl trap::Label for DeclarationTrapLabel { + fn as_untyped(&self) -> trap::UntypedLabel { + self.0 + } +} + +impl From for trap::Arg { + fn from(value: DeclarationTrapLabel) -> Self { + value.0.into() + } +} + +#[derive(Debug)] +pub struct Declaration { + pub id: trap::TrapId, +} + +impl trap::TrapEntry for Declaration { + fn class_name() -> &'static str { "Declaration" } + + fn extract_id(&mut self) -> trap::TrapId { + std::mem::replace(&mut self.id, trap::TrapId::Star) + } + + fn emit(self, id: Self::Label, out: &mut trap::Writer) { + out.add_tuple("declarations", vec![id.into()]); + } +} + +impl trap::TrapClass for Declaration { + type Label = DeclarationTrapLabel; +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub struct ExprTrapLabel(trap::UntypedLabel); + +impl From for ExprTrapLabel { + fn from(value: trap::UntypedLabel) -> Self { + Self(value) + } +} + +impl From for trap::TrapId { + fn from(value: ExprTrapLabel) -> Self { + Self::Label(value) + } +} + +impl trap::Label for ExprTrapLabel { + fn as_untyped(&self) -> trap::UntypedLabel { + self.0 + } +} + +impl From for trap::Arg { + fn from(value: ExprTrapLabel) -> Self { + value.0.into() + } +} + +#[derive(Debug)] +pub struct Expr { + pub id: trap::TrapId, +} + +impl trap::TrapEntry for Expr { + fn class_name() -> &'static str { "Expr" } + + fn extract_id(&mut self) -> trap::TrapId { + std::mem::replace(&mut self.id, trap::TrapId::Star) + } + + fn emit(self, id: Self::Label, out: &mut trap::Writer) { + out.add_tuple("exprs", vec![id.into()]); + } +} + +impl trap::TrapClass for Expr { + type Label = ExprTrapLabel; +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub struct LabelTrapLabel(trap::UntypedLabel); + +impl From for LabelTrapLabel { + fn from(value: trap::UntypedLabel) -> Self { + Self(value) + } +} + +impl From for trap::TrapId