From 4f0fe1ce3a6424caa1ca9349e2896501c1e3d748 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 30 Aug 2024 13:05:46 +0200 Subject: [PATCH] Rust: bazel packaging --- MODULE.bazel | 14 +++++ misc/codegen/generators/rustgen.py | 13 +++-- misc/codegen/lib/rust.py | 10 +++- misc/codegen/templates/rust_classes.mustache | 4 +- rust/.idea/.gitignore | 5 ++ rust/.idea/misc.xml | 6 ++ rust/.idea/modules.xml | 8 +++ rust/.idea/rust.iml | 13 +++++ rust/.idea/vcs.xml | 6 ++ rust/BUILD.bazel | 59 ++++++++++++++++++-- rust/codegen.conf | 2 +- rust/codegen/BUILD.bazel | 6 +- rust/codeql-extractor.yml | 15 +++++ rust/{ => extractor}/.gitignore | 0 rust/extractor/BUILD.bazel | 15 +++++ rust/{ => extractor}/Cargo.lock | 3 +- rust/{ => extractor}/Cargo.toml | 5 ++ rust/{ => extractor}/src/archive.rs | 2 +- rust/{ => extractor}/src/config.rs | 0 rust/{ => extractor}/src/generated/mod.rs | 0 rust/{ => extractor}/src/generated/top.rs | 10 ++-- rust/{ => extractor}/src/main.rs | 2 - rust/{ => extractor}/src/path.rs | 0 rust/{ => extractor}/src/translate.rs | 0 rust/{ => extractor}/src/trap.rs | 7 ++- rust/ql/lib/rust.dbscheme.stats | 4 ++ rust/tools/BUILD.bazel | 9 +++ rust/tools/index.sh | 17 ++++++ 28 files changed, 207 insertions(+), 28 deletions(-) create mode 100644 rust/.idea/.gitignore create mode 100644 rust/.idea/misc.xml create mode 100644 rust/.idea/modules.xml create mode 100644 rust/.idea/rust.iml create mode 100644 rust/.idea/vcs.xml create mode 100644 rust/codeql-extractor.yml rename rust/{ => extractor}/.gitignore (100%) create mode 100644 rust/extractor/BUILD.bazel rename rust/{ => extractor}/Cargo.lock (99%) rename rust/{ => extractor}/Cargo.toml (67%) rename rust/{ => extractor}/src/archive.rs (94%) rename rust/{ => extractor}/src/config.rs (100%) rename rust/{ => extractor}/src/generated/mod.rs (100%) rename rust/{ => extractor}/src/generated/top.rs (82%) rename rust/{ => extractor}/src/main.rs (98%) rename rust/{ => extractor}/src/path.rs (100%) rename rust/{ => extractor}/src/translate.rs (100%) rename rust/{ => extractor}/src/trap.rs (95%) create mode 100644 rust/ql/lib/rust.dbscheme.stats create mode 100644 rust/tools/BUILD.bazel create mode 100755 rust/tools/index.sh diff --git a/MODULE.bazel b/MODULE.bazel index d2add26c88d..783c35dc5fc 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -63,6 +63,20 @@ r.from_cargo( ) use_repo(r, ruby_deps = "rd") +rsp = use_extension( + "@rules_rust//crate_universe:extension.bzl", + "crate", + isolate = True, +) +rsp.from_cargo( + name = "rs_deps", + cargo_lockfile = "//rust/extractor:Cargo.lock", + manifests = [ + "//rust/extractor:Cargo.toml", + ], +) +use_repo(rsp, rust_deps = "rs_deps") + dotnet = use_extension("@rules_dotnet//dotnet:extensions.bzl", "dotnet") dotnet.toolchain(dotnet_version = "8.0.101") use_repo(dotnet, "dotnet_toolchains") diff --git a/misc/codegen/generators/rustgen.py b/misc/codegen/generators/rustgen.py index dc5494aa731..04a6538998f 100644 --- a/misc/codegen/generators/rustgen.py +++ b/misc/codegen/generators/rustgen.py @@ -26,7 +26,7 @@ def _get_type(t: str) -> str: def _get_field(cls: schema.Class, p: schema.Property) -> rust.Field: - table_name = None + table_name = inflection.tableize(cls.name) if not p.is_single: table_name = f"{cls.name}_{p.name}" if p.is_predicate: @@ -47,11 +47,12 @@ def _get_field(cls: schema.Class, p: schema.Property) -> rust.Field: def _get_properties( - cls: schema.Class, lookup: dict[str, schema.Class] -) -> typing.Iterable[schema.Property]: + cls: schema.Class, lookup: dict[str, schema.Class], +) -> typing.Iterable[tuple[schema.Class, schema.Property]]: for b in cls.bases: yield from _get_properties(lookup[b], lookup) - yield from cls.properties + for p in cls.properties: + yield cls, p class Processor: @@ -63,8 +64,8 @@ class Processor: return rust.Class( name=name, fields=[ - _get_field(cls, p) - for p in _get_properties(cls, self._classmap) + _get_field(c, p) + for c, p in _get_properties(cls, self._classmap) if "rust_skip" not in p.pragmas and not p.synth ], table_name=inflection.tableize(cls.name), diff --git a/misc/codegen/lib/rust.py b/misc/codegen/lib/rust.py index 0c51d33fbf2..5e6405759b8 100644 --- a/misc/codegen/lib/rust.py +++ b/misc/codegen/lib/rust.py @@ -78,7 +78,7 @@ def get_field_override(field: str): class Field: field_name: str base_type: str - table_name: str = None + table_name: str is_optional: bool = False is_repeated: bool = False is_unordered: bool = False @@ -121,8 +121,12 @@ class Class: fields: list[Field] = dataclasses.field(default_factory=list) @property - def single_fields(self): - return [f for f in self.fields if f.is_single] + def single_field_entries(self): + ret = {self.table_name: []} + for f in self.fields: + if f.is_single: + ret.setdefault(f.table_name, []).append(f) + return [{"table_name": k, "fields": v} for k, v in ret.items()] @dataclasses.dataclass diff --git a/misc/codegen/templates/rust_classes.mustache b/misc/codegen/templates/rust_classes.mustache index 5102b8aea40..a2c0aff6db8 100644 --- a/misc/codegen/templates/rust_classes.mustache +++ b/misc/codegen/templates/rust_classes.mustache @@ -18,7 +18,9 @@ impl TrapEntry for {{name}} { } fn emit(self, id: TrapLabel, out: &mut W) -> std::io::Result<()> { - write!(out, "{{table_name}}({id}{{#single_fields}}, {}{{/single_fields}})\n"{{#single_fields}}, {{#emitter}}self.{{field_name}}{{/emitter}}{{/single_fields}})?; + {{#single_field_entries}} + write!(out, "{{table_name}}({id}{{#fields}}, {}{{/fields}})\n"{{#fields}}, {{#emitter}}self.{{field_name}}{{/emitter}}{{/fields}})?; + {{/single_field_entries}} {{#fields}} {{#is_predicate}} if self.{{field_name}} { diff --git a/rust/.idea/.gitignore b/rust/.idea/.gitignore new file mode 100644 index 00000000000..b58b603fea7 --- /dev/null +++ b/rust/.idea/.gitignore @@ -0,0 +1,5 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Editor-based HTTP Client requests +/httpRequests/ diff --git a/rust/.idea/misc.xml b/rust/.idea/misc.xml new file mode 100644 index 00000000000..73797bfcc85 --- /dev/null +++ b/rust/.idea/misc.xml @@ -0,0 +1,6 @@ + + + + + diff --git a/rust/.idea/modules.xml b/rust/.idea/modules.xml new file mode 100644 index 00000000000..9996827f2ba --- /dev/null +++ b/rust/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/rust/.idea/rust.iml b/rust/.idea/rust.iml new file mode 100644 index 00000000000..fd100f63146 --- /dev/null +++ b/rust/.idea/rust.iml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + diff --git a/rust/.idea/vcs.xml b/rust/.idea/vcs.xml new file mode 100644 index 00000000000..54e4b961ee0 --- /dev/null +++ b/rust/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + diff --git a/rust/BUILD.bazel b/rust/BUILD.bazel index 87aedadba29..606e0e70e9c 100644 --- a/rust/BUILD.bazel +++ b/rust/BUILD.bazel @@ -1,4 +1,55 @@ -exports_files([ - "codegen.conf", - "schema.py", -]) +load("@rules_pkg//pkg:mappings.bzl", "pkg_filegroup") +load( + "//misc/bazel:pkg.bzl", + "codeql_pack", + "codeql_pkg_files", +) + +package(default_visibility = ["//rust:__subpackages__"]) + +filegroup( + name = "schema", + srcs = ["schema.py"], +) + +filegroup( + name = "schema-includes", + srcs = glob(["*.dbscheme"]), +) + +filegroup( + name = "codegen-conf", + srcs = ["codegen.conf"], +) + +codeql_pkg_files( + name = "tools-arch", + exes = ["//rust/extractor"], + prefix = "{CODEQL_PLATFORM}", +) + +pkg_filegroup( + name = "tools", + srcs = [ + ":tools-arch", + "//rust/tools", + ], + prefix = "tools", +) + +codeql_pkg_files( + name = "root-files", + srcs = [ + "codeql-extractor.yml", + "ql/lib/rust.dbscheme", + "ql/lib/rust.dbscheme.stats", + ], +) + +codeql_pack( + name = "rust", + srcs = [ + ":root-files", + ":tools", + ], +) diff --git a/rust/codegen.conf b/rust/codegen.conf index df2f6270a9a..6e8a59de9ae 100644 --- a/rust/codegen.conf +++ b/rust/codegen.conf @@ -4,6 +4,6 @@ --ql-output=ql/lib/codeql/rust/generated --ql-stub-output=ql/lib/codeql/rust/elements --ql-test-output=ql/test/extractor-tests/generated ---rust-output=src/generated +--rust-output=extractor/src/generated --generated-registry=.generated.list --script-name=codegen diff --git a/rust/codegen/BUILD.bazel b/rust/codegen/BUILD.bazel index 82ac81abab0..99f737538a6 100644 --- a/rust/codegen/BUILD.bazel +++ b/rust/codegen/BUILD.bazel @@ -5,11 +5,11 @@ native_binary( src = "//misc/codegen", out = "codegen", args = [ - "--configuration-file=$(location //rust:codegen.conf)", + "--configuration-file=$(location //rust:codegen-conf)", ], data = [ - "//rust:codegen.conf", - "//rust:schema.py", + "//rust:codegen-conf", + "//rust:schema", ], visibility = ["//rust:__subpackages__"], ) diff --git a/rust/codeql-extractor.yml b/rust/codeql-extractor.yml new file mode 100644 index 00000000000..af7d4475867 --- /dev/null +++ b/rust/codeql-extractor.yml @@ -0,0 +1,15 @@ +name: "rust" +display_name: "Rust" +version: 0.1.0 +column_kind: "utf8" +build_modes: + - none +github_api_languages: + - Rust +scc_languages: + - Rust +file_types: + - name: rust + display_name: Rust files + extensions: + - .rs diff --git a/rust/.gitignore b/rust/extractor/.gitignore similarity index 100% rename from rust/.gitignore rename to rust/extractor/.gitignore diff --git a/rust/extractor/BUILD.bazel b/rust/extractor/BUILD.bazel new file mode 100644 index 00000000000..ffe1ee92de9 --- /dev/null +++ b/rust/extractor/BUILD.bazel @@ -0,0 +1,15 @@ +load("@rust_deps//:defs.bzl", "aliases", "all_crate_deps") +load("//misc/bazel:rust.bzl", "codeql_rust_binary") + +codeql_rust_binary( + name = "extractor", + srcs = glob(["src/**/*.rs"]), + aliases = aliases(), + proc_macro_deps = all_crate_deps( + proc_macro = True, + ), + visibility = ["//rust:__subpackages__"], + deps = all_crate_deps( + normal = True, + ), +) diff --git a/rust/Cargo.lock b/rust/extractor/Cargo.lock similarity index 99% rename from rust/Cargo.lock rename to rust/extractor/Cargo.lock index 35f700fa120..98a3f590e2a 100644 --- a/rust/Cargo.lock +++ b/rust/extractor/Cargo.lock @@ -1566,8 +1566,7 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" [[package]] name = "rustc_apfloat" version = "0.2.1+llvm-462a31f5a5ab" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "886d94c63c812a8037c4faca2607453a0fa4cf82f734665266876b022244543f" +source = "git+https://github.com/redsun82/rustc_apfloat.git?rev=096d585100636bc2e9f09d7eefec38c5b334d47b#096d585100636bc2e9f09d7eefec38c5b334d47b" dependencies = [ "bitflags 1.3.2", "smallvec", diff --git a/rust/Cargo.toml b/rust/extractor/Cargo.toml similarity index 67% rename from rust/Cargo.toml rename to rust/extractor/Cargo.toml index 716a506edc6..8c8ef26ef2b 100644 --- a/rust/Cargo.toml +++ b/rust/extractor/Cargo.toml @@ -21,3 +21,8 @@ serde = "1.0.209" serde_with = "3.9.0" stderrlog = "0.6.0" triomphe = "0.1.13" + +[patch.crates-io] +# patch for build script bug preventing bazel build +# see https://github.com/rust-lang/rustc_apfloat/pull/17 +rustc_apfloat = { git = "https://github.com/redsun82/rustc_apfloat.git", rev = "096d585100636bc2e9f09d7eefec38c5b334d47b" } diff --git a/rust/src/archive.rs b/rust/extractor/src/archive.rs similarity index 94% rename from rust/src/archive.rs rename to rust/extractor/src/archive.rs index 09847ac85d6..68a652543c3 100644 --- a/rust/src/archive.rs +++ b/rust/extractor/src/archive.rs @@ -19,7 +19,7 @@ impl Archiver { let mut dest = self.root.clone(); dest.push(path::key(source)); let parent = dest.parent().unwrap(); - if fs::exists(&dest)? { + if fs::metadata(&dest).is_ok() { return Ok(()) } fs::create_dir_all(parent)?; diff --git a/rust/src/config.rs b/rust/extractor/src/config.rs similarity index 100% rename from rust/src/config.rs rename to rust/extractor/src/config.rs diff --git a/rust/src/generated/mod.rs b/rust/extractor/src/generated/mod.rs similarity index 100% rename from rust/src/generated/mod.rs rename to rust/extractor/src/generated/mod.rs diff --git a/rust/src/generated/top.rs b/rust/extractor/src/generated/top.rs similarity index 82% rename from rust/src/generated/top.rs rename to rust/extractor/src/generated/top.rs index d58970b23bc..d337d08c9d4 100644 --- a/rust/src/generated/top.rs +++ b/rust/extractor/src/generated/top.rs @@ -15,7 +15,8 @@ impl TrapEntry for DbFile { } fn emit(self, id: TrapLabel, out: &mut W) -> std::io::Result<()> { - write!(out, "db_files({id}, {})\n", quoted(&self.name))?; + write!(out, "db_files({id})\n")?; + write!(out, "files({id}, {})\n", quoted(&self.name))?; Ok(()) } } @@ -36,7 +37,8 @@ impl TrapEntry for DbLocation { } fn emit(self, id: TrapLabel, out: &mut W) -> std::io::Result<()> { - write!(out, "db_locations({id}, {}, {}, {}, {}, {})\n", self.file, self.start_line, self.start_column, self.end_line, self.end_column)?; + write!(out, "db_locations({id})\n")?; + write!(out, "locations({id}, {}, {}, {}, {}, {})\n", self.file, self.start_line, self.start_column, self.end_line, self.end_column)?; Ok(()) } } @@ -56,7 +58,7 @@ impl TrapEntry for Function { fn emit(self, id: TrapLabel, out: &mut W) -> std::io::Result<()> { write!(out, "functions({id}, {})\n", quoted(&self.name))?; if let Some(ref v) = &self.location { - write!(out, "function_locations({id}, {})\n", v)?; + write!(out, "locatable_locations({id}, {})\n", v)?; } Ok(()) } @@ -77,7 +79,7 @@ impl TrapEntry for Module { fn emit(self, id: TrapLabel, out: &mut W) -> std::io::Result<()> { write!(out, "modules({id})\n")?; if let Some(ref v) = &self.location { - write!(out, "module_locations({id}, {})\n", v)?; + write!(out, "locatable_locations({id}, {})\n", v)?; } for (i, &ref v) in self.declarations.iter().enumerate() { write!(out, "module_declarations({id}, {}, {})\n", i, v)?; diff --git a/rust/src/main.rs b/rust/extractor/src/main.rs similarity index 98% rename from rust/src/main.rs rename to rust/extractor/src/main.rs index 016c948739b..cb8d3f2deb2 100644 --- a/rust/src/main.rs +++ b/rust/extractor/src/main.rs @@ -1,5 +1,3 @@ -#![feature(path_add_extension)] - use std::fs; use std::path::{PathBuf, Path}; use ra_ap_project_model::CargoConfig; diff --git a/rust/src/path.rs b/rust/extractor/src/path.rs similarity index 100% rename from rust/src/path.rs rename to rust/extractor/src/path.rs diff --git a/rust/src/translate.rs b/rust/extractor/src/translate.rs similarity index 100% rename from rust/src/translate.rs rename to rust/extractor/src/translate.rs diff --git a/rust/src/trap.rs b/rust/extractor/src/trap.rs similarity index 95% rename from rust/src/trap.rs rename to rust/extractor/src/trap.rs index d74f252577a..ea5be477370 100644 --- a/rust/src/trap.rs +++ b/rust/extractor/src/trap.rs @@ -1,3 +1,4 @@ +use std::ffi::OsString; use std::fmt::{Debug, Display, Formatter}; use std::fs::File; use std::io::Write; @@ -158,7 +159,11 @@ impl TrapFileProvider { pub fn create(&self, category: &str, key: &Path) -> std::io::Result { let mut path = PathBuf::from(category); path.push(path::key(key)); - path.add_extension("trap"); + path.set_extension(path.extension().map(|e| { + let mut o : OsString = e.to_owned(); + o.push(".trap"); + o + }).unwrap_or("trap".into())); let trap_name = String::from(path.to_string_lossy()); debug!("creating trap file {}", trap_name); path = self.trap_dir.join(path); diff --git a/rust/ql/lib/rust.dbscheme.stats b/rust/ql/lib/rust.dbscheme.stats new file mode 100644 index 00000000000..9995467e33e --- /dev/null +++ b/rust/ql/lib/rust.dbscheme.stats @@ -0,0 +1,4 @@ + + + + diff --git a/rust/tools/BUILD.bazel b/rust/tools/BUILD.bazel new file mode 100644 index 00000000000..27bc5bbe600 --- /dev/null +++ b/rust/tools/BUILD.bazel @@ -0,0 +1,9 @@ +load("//misc/bazel:pkg.bzl", "codeql_pkg_files") + +codeql_pkg_files( + name = "tools", + exes = [ + "index.sh", + ], + visibility = ["//rust:__pkg__"], +) diff --git a/rust/tools/index.sh b/rust/tools/index.sh new file mode 100755 index 00000000000..7035f09d0a4 --- /dev/null +++ b/rust/tools/index.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# TODO move this to rust code + +inputs=($(find -name Cargo.toml)) + +if [ "${#inputs}" -eq 0 ]; then + inputs=($(find -name rust-project.json)) + if [ "${#inputs}" -eq 0 ]; then + inputs=($(find -name '*.rs')) + if [ "${#inputs}" -eq 0 ]; then + echo "no source files found" >&2 + exit 1 + fi + fi +fi + +exec "$CODEQL_EXTRACTOR_RUST_ROOT/tools/$CODEQL_PLATFORM/extractor" "${inputs[@]}"