hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2025-12-16 16:53:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

Rust: reuse shared rust trap library

Browse Source
This commit is contained in:
Paolo Tranquilli
2024-08-30 16:08:37 +02:00
parent 4f0fe1ce3a
commit 7e1290aa74
19 changed files with 677 additions and 216 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.pre-commit-config.yaml
View File

@@ -45,7 +45,7 @@ repos:
- id: sync-files - id: sync-files
name: Fix files required to be identical name: Fix files required to be identical
files: \.(qll?|qhelp|swift)$|^config/identical-files\.json$ files: \.(qll?|qhelp|swift|toml)$|^config/identical-files\.json$
language: system language: system
entry: python3 config/sync-files.py --latest entry: python3 config/sync-files.py --latest
pass_filenames: false pass_filenames: false

3
config/identical-files.json
View File

@@ -358,6 +358,7 @@
], ],
"shared tree-sitter extractor cargo.toml": [ "shared tree-sitter extractor cargo.toml": [
"shared/tree-sitter-extractor/Cargo.toml", "shared/tree-sitter-extractor/Cargo.toml",
"ruby/extractor/codeql-extractor-fake-crate/Cargo.toml" "ruby/extractor/codeql-extractor-fake-crate/Cargo.toml",
"rust/extractor/codeql-extractor-fake-crate/Cargo.toml"
] ]
} }

8
misc/codegen/generators/rustgen.py
View File

@@ -13,14 +13,16 @@ from misc.codegen.loaders import schemaloader
def _get_type(t: str) -> str: def _get_type(t: str) -> str:
match t: match t:
case None | "boolean": # None means a predicate case None: # None means a predicate
return "bool" return "bool"
case "string": case "string":
return "String" return "String"
case "int": case "int":
return "i32" return "usize"
case _ if t[0].isupper(): case _ if t[0].isupper():
return "TrapLabel" return "trap::Label"
case "boolean":
assert False, "boolean unsupported"
case _: case _:
return t return t

13
misc/codegen/lib/rust.py
View File

@@ -58,10 +58,6 @@ keywords = {
} }
_field_overrides = [ _field_overrides = [
(
re.compile(r"(start|end)_(line|column)|(.*_)?index|width|num_.*"),
{"base_type": "u32"},
),
(re.compile(r"(.*)_"), lambda m: {"field_name": m[1]}), (re.compile(r"(.*)_"), lambda m: {"field_name": m[1]}),
] ]
@@ -98,20 +94,13 @@ class Field:
type = f"Vec<{type}>" type = f"Vec<{type}>"
return type return type
# using @property breaks pystache internals here
def emitter(self):
if self.type == "String":
return lambda x: f"quoted(&{x})"
else:
return lambda x: x
@property @property
def is_single(self): def is_single(self):
return not (self.is_optional or self.is_repeated or self.is_predicate) return not (self.is_optional or self.is_repeated or self.is_predicate)
@property @property
def is_label(self): def is_label(self):
return self.base_type == "TrapLabel" return self.base_type == "trap::Label"
@dataclasses.dataclass @dataclasses.dataclass

23
misc/codegen/templates/rust_classes.mustache
View File

@@ -1,7 +1,7 @@
// generated by {{generator}} // generated by {{generator}}
use crate::trap::{TrapLabel, TrapId, TrapEntry, quoted}; use crate::trap::{TrapId, TrapEntry};
use std::io::Write; use codeql_extractor::trap;
{{#classes}} {{#classes}}
#[derive(Debug)] #[derive(Debug)]
@@ -17,37 +17,36 @@ impl TrapEntry for {{name}} {
std::mem::replace(&mut self.id, TrapId::Star) std::mem::replace(&mut self.id, TrapId::Star)
} }
fn emit<W: Write>(self, id: TrapLabel, out: &mut W) -> std::io::Result<()> { fn emit(self, id: trap::Label, out: &mut trap::Writer) {
{{#single_field_entries}} {{#single_field_entries}}
write!(out, "{{table_name}}({id}{{#fields}}, {}{{/fields}})\n"{{#fields}}, {{#emitter}}self.{{field_name}}{{/emitter}}{{/fields}})?; out.add_tuple("{{table_name}}", vec![trap::Arg::Label(id){{#fields}}, self.{{field_name}}.into(){{/fields}}]);
{{/single_field_entries}} {{/single_field_entries}}
{{#fields}} {{#fields}}
{{#is_predicate}} {{#is_predicate}}
if self.{{field_name}} { if self.{{field_name}} {
write!(out, "{{table_name}}({id})\n")?; out.add_tuple("{{table_name}}", vec![trap::Arg::Label(id)]);
} }
{{/is_predicate}} {{/is_predicate}}
{{#is_optional}} {{#is_optional}}
{{^is_repeated}} {{^is_repeated}}
if let Some(ref v) = &self.{{field_name}} { if let Some(v) = self.{{field_name}} {
write!(out, "{{table_name}}({id}, {})\n", {{#emitter}}v{{/emitter}})?; out.add_tuple("{{table_name}}", vec![trap::Arg::Label(id), v.into()]);
} }
{{/is_repeated}} {{/is_repeated}}
{{/is_optional}} {{/is_optional}}
{{#is_repeated}} {{#is_repeated}}
for (i, &ref v) in self.{{field_name}}.iter().enumerate() { for (i, &v) in self.{{field_name}}.iter().enumerate() {
{{^is_optional}} {{^is_optional}}
write!(out, "{{table_name}}({id}, {{^is_unordered}}{}, {{/is_unordered}}{})\n", {{^is_unordered}}i, {{/is_unordered}}{{#emitter}}v{{/emitter}})?; out.add_tuple("{{table_name}}", vec![trap::Arg::Label(id){{^is_unordered}}, i.into(){{/is_unordered}}, v.into()]);
{{/is_optional}} {{/is_optional}}
{{#is_optional}} {{#is_optional}}
if let Some(ref vv) = &v { if let Some(vv) = v {
write!(out, "{{table_name}}({id}, {{^is_unordered}}{}, {{/is_unordered}}{})\n", {{^is_unordered}}i, {{/is_unordered}}{{#emitter}}vv{{/emitter}})?; out.add_tuple("{{table_name}}", vec![trap::Arg::Label(id){{^is_unordered}}, i.into(){{/is_unordered}}, v.into()]);
} }
{{/is_optional}} {{/is_optional}}
} }
{{/is_repeated}} {{/is_repeated}}
{{/fields}} {{/fields}}
Ok(())
} }
} }
{{/classes}} {{/classes}}

1
rust/extractor/.cargo/config.toml Normal file
View File

@@ -0,0 +1 @@
paths = ["../../shared/tree-sitter-extractor"]

4
rust/extractor/BUILD.bazel
View File

@@ -11,5 +11,7 @@ codeql_rust_binary(
visibility = ["//rust:__subpackages__"], visibility = ["//rust:__subpackages__"],
deps = all_crate_deps( deps = all_crate_deps(
normal = True, normal = True,
), ) + [
"//shared/tree-sitter-extractor:codeql-extractor",
],
) )

418
rust/extractor/Cargo.lock generated
View File

@@ -2,6 +2,21 @@
# It is not intended for manual editing. # It is not intended for manual editing.
version = 3 version = 3
[[package]]
name = "adler2"
version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627"
[[package]]
name = "aho-corasick"
version = "1.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916"
dependencies = [
"memchr",
]
[[package]] [[package]]
name = "always-assert" name = "always-assert"
version = "0.2.0" version = "0.2.0"
@@ -120,6 +135,16 @@ version = "2.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de"
[[package]]
name = "bstr"
version = "1.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "40723b8fb387abc38f4f4a37c09073622e41dd12327033091ef8950659e6dc0c"
dependencies = [
"memchr",
"serde",
]
[[package]] [[package]]
name = "bumpalo" name = "bumpalo"
version = "3.16.0" version = "3.16.0"
@@ -132,6 +157,12 @@ version = "1.17.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "773d90827bc3feecfb67fab12e24de0749aad83c74b9504ecde46237b5cd24e2" checksum = "773d90827bc3feecfb67fab12e24de0749aad83c74b9504ecde46237b5cd24e2"
[[package]]
name = "byteorder"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
[[package]] [[package]]
name = "camino" name = "camino"
version = "1.1.9" version = "1.1.9"
@@ -285,12 +316,35 @@ version = "0.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97"
[[package]]
name = "codeql-extractor"
version = "0.2.0"
dependencies = [
"chrono",
"encoding",
"flate2",
"globset",
"lazy_static",
"num_cpus",
"rand",
"rayon",
"regex",
"serde",
"serde_json",
"tracing",
"tracing-subscriber",
"tree-sitter",
"tree-sitter-json",
"tree-sitter-ql",
]
[[package]] [[package]]
name = "codeql-rust" name = "codeql-rust"
version = "0.1.0" version = "0.1.0"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"clap", "clap",
"codeql-extractor",
"figment", "figment",
"log", "log",
"num-traits", "num-traits",
@@ -332,6 +386,15 @@ version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0570650661aa447e7335f1d5e4f499d8e58796e617bedc9267d971e51c8b49d4" checksum = "0570650661aa447e7335f1d5e4f499d8e58796e617bedc9267d971e51c8b49d4"
[[package]]
name = "crc32fast"
version = "1.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3"
dependencies = [
"cfg-if",
]
[[package]] [[package]]
name = "crossbeam-channel" name = "crossbeam-channel"
version = "0.5.13" version = "0.5.13"
@@ -445,6 +508,70 @@ dependencies = [
"log", "log",
] ]
[[package]]
name = "encoding"
version = "0.2.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6b0d943856b990d12d3b55b359144ff341533e516d94098b1d3fc1ac666d36ec"
dependencies = [
"encoding-index-japanese",
"encoding-index-korean",
"encoding-index-simpchinese",
"encoding-index-singlebyte",
"encoding-index-tradchinese",
]
[[package]]
name = "encoding-index-japanese"
version = "1.20141219.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "04e8b2ff42e9a05335dbf8b5c6f7567e5591d0d916ccef4e0b1710d32a0d0c91"
dependencies = [
"encoding_index_tests",
]
[[package]]
name = "encoding-index-korean"
version = "1.20141219.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4dc33fb8e6bcba213fe2f14275f0963fd16f0a02c878e3095ecfdf5bee529d81"
dependencies = [
"encoding_index_tests",
]
[[package]]
name = "encoding-index-simpchinese"
version = "1.20141219.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d87a7194909b9118fc707194baa434a4e3b0fb6a5a757c73c3adb07aa25031f7"
dependencies = [
"encoding_index_tests",
]
[[package]]
name = "encoding-index-singlebyte"
version = "1.20141219.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3351d5acffb224af9ca265f435b859c7c01537c0849754d3db3fdf2bfe2ae84a"
dependencies = [
"encoding_index_tests",
]
[[package]]
name = "encoding-index-tradchinese"
version = "1.20141219.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fd0e20d5688ce3cab59eb3ef3a2083a5c77bf496cb798dc6fcdb75f323890c18"
dependencies = [
"encoding_index_tests",
]
[[package]]
name = "encoding_index_tests"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a246d82be1c9d791c5dfde9a2bd045fc3cbba3fa2b11ad558f27d01712f00569"
[[package]] [[package]]
name = "equivalent" name = "equivalent"
version = "1.0.1" version = "1.0.1"
@@ -482,6 +609,16 @@ version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80"
[[package]]
name = "flate2"
version = "1.0.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "324a1be68054ef05ad64b861cc9eaf1d623d2d8cb25b4bf2cb9cdd902b4bf253"
dependencies = [
"crc32fast",
"miniz_oxide",
]
[[package]] [[package]]
name = "fnv" name = "fnv"
version = "1.0.7" version = "1.0.7"
@@ -503,6 +640,30 @@ version = "0.4.7"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ab85b9b05e3978cc9a9cf8fea7f01b494e1a09ed3037e16ba39edc7a29eb61a" checksum = "7ab85b9b05e3978cc9a9cf8fea7f01b494e1a09ed3037e16ba39edc7a29eb61a"
[[package]]
name = "getrandom"
version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7"
dependencies = [
"cfg-if",
"libc",
"wasi",
]
[[package]]
name = "globset"
version = "0.4.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "57da3b9b5b85bd66f31093f8c408b90a74431672542466497dcbdfdc02034be1"
dependencies = [
"aho-corasick",
"bstr",
"log",
"regex-automata 0.4.7",
"regex-syntax 0.8.4",
]
[[package]] [[package]]
name = "hashbrown" name = "hashbrown"
version = "0.12.3" version = "0.12.3"
@@ -527,6 +688,12 @@ version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
[[package]]
name = "hermit-abi"
version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024"
[[package]] [[package]]
name = "hermit-abi" name = "hermit-abi"
version = "0.4.0" version = "0.4.0"
@@ -631,7 +798,7 @@ version = "0.4.13"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "261f68e344040fbd0edea105bef17c66edf46f984ddb1115b775ce31be948f4b" checksum = "261f68e344040fbd0edea105bef17c66edf46f984ddb1115b775ce31be948f4b"
dependencies = [ dependencies = [
"hermit-abi", "hermit-abi 0.4.0",
"libc", "libc",
"windows-sys 0.52.0", "windows-sys 0.52.0",
] ]
@@ -698,6 +865,12 @@ version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3752f229dcc5a481d60f385fa479ff46818033d881d2d801aa27dffcfb5e8306" checksum = "3752f229dcc5a481d60f385fa479ff46818033d881d2d801aa27dffcfb5e8306"
[[package]]
name = "lazy_static"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
[[package]] [[package]]
name = "libc" name = "libc"
version = "0.2.158" version = "0.2.158"
@@ -747,6 +920,15 @@ version = "0.11.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "75761162ae2b0e580d7e7c390558127e5f01b4194debd6221fd8c207fc80e3f5" checksum = "75761162ae2b0e580d7e7c390558127e5f01b4194debd6221fd8c207fc80e3f5"
[[package]]
name = "matchers"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558"
dependencies = [
"regex-automata 0.1.10",
]
[[package]] [[package]]
name = "memchr" name = "memchr"
version = "2.7.4" version = "2.7.4"
@@ -762,6 +944,15 @@ dependencies = [
"autocfg", "autocfg",
] ]
[[package]]
name = "miniz_oxide"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1"
dependencies = [
"adler2",
]
[[package]] [[package]]
name = "mio" name = "mio"
version = "0.8.11" version = "0.8.11"
@@ -808,6 +999,16 @@ dependencies = [
"windows-sys 0.48.0", "windows-sys 0.48.0",
] ]
[[package]]
name = "nu-ansi-term"
version = "0.46.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84"
dependencies = [
"overload",
"winapi",
]
[[package]] [[package]]
name = "num-conv" name = "num-conv"
version = "0.1.0" version = "0.1.0"
@@ -823,6 +1024,16 @@ dependencies = [
"autocfg", "autocfg",
] ]
[[package]]
name = "num_cpus"
version = "1.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43"
dependencies = [
"hermit-abi 0.3.9",
"libc",
]
[[package]] [[package]]
name = "once_cell" name = "once_cell"
version = "1.19.0" version = "1.19.0"
@@ -835,6 +1046,12 @@ version = "11.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b410bbe7e14ab526a0e86877eb47c6996a2bd7746f027ba551028c925390e4e9" checksum = "b410bbe7e14ab526a0e86877eb47c6996a2bd7746f027ba551028c925390e4e9"
[[package]]
name = "overload"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39"
[[package]] [[package]]
name = "parking_lot" name = "parking_lot"
version = "0.12.3" version = "0.12.3"
@@ -922,6 +1139,15 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
[[package]]
name = "ppv-lite86"
version = "0.2.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04"
dependencies = [
"zerocopy",
]
[[package]] [[package]]
name = "proc-macro2" name = "proc-macro2"
version = "1.0.86" version = "1.0.86"
@@ -1515,6 +1741,36 @@ dependencies = [
"walkdir", "walkdir",
] ]
[[package]]
name = "rand"
version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
dependencies = [
"libc",
"rand_chacha",
"rand_core",
]
[[package]]
name = "rand_chacha"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
dependencies = [
"ppv-lite86",
"rand_core",
]
[[package]]
name = "rand_core"
version = "0.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
dependencies = [
"getrandom",
]
[[package]] [[package]]
name = "rayon" name = "rayon"
version = "1.10.0" version = "1.10.0"
@@ -1544,6 +1800,50 @@ dependencies = [
"bitflags 2.6.0", "bitflags 2.6.0",
] ]
[[package]]
name = "regex"
version = "1.10.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619"
dependencies = [
"aho-corasick",
"memchr",
"regex-automata 0.4.7",
"regex-syntax 0.8.4",
]
[[package]]
name = "regex-automata"
version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132"
dependencies = [
"regex-syntax 0.6.29",
]
[[package]]
name = "regex-automata"
version = "0.4.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df"
dependencies = [
"aho-corasick",
"memchr",
"regex-syntax 0.8.4",
]
[[package]]
name = "regex-syntax"
version = "0.6.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1"
[[package]]
name = "regex-syntax"
version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b"
[[package]] [[package]]
name = "rowan" name = "rowan"
version = "0.15.15" version = "0.15.15"
@@ -1670,6 +1970,15 @@ dependencies = [
"syn", "syn",
] ]
[[package]]
name = "sharded-slab"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6"
dependencies = [
"lazy_static",
]
[[package]] [[package]]
name = "shlex" name = "shlex"
version = "1.3.0" version = "1.3.0"
@@ -1849,6 +2158,64 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54"
dependencies = [ dependencies = [
"once_cell", "once_cell",
"valuable",
]
[[package]]
name = "tracing-log"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3"
dependencies = [
"log",
"once_cell",
"tracing-core",
]
[[package]]
name = "tracing-subscriber"
version = "0.3.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b"
dependencies = [
"matchers",
"nu-ansi-term",
"once_cell",
"regex",
"sharded-slab",
"smallvec",
"thread_local",
"tracing",
"tracing-core",
"tracing-log",
]
[[package]]
name = "tree-sitter"
version = "0.22.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df7cc499ceadd4dcdf7ec6d4cbc34ece92c3fa07821e287aedecd4416c516dca"
dependencies = [
"cc",
"regex",
]
[[package]]
name = "tree-sitter-json"
version = "0.21.0"
source = "git+https://github.com/tree-sitter/tree-sitter-json#bdd69eb8c8a58a9f54df03de0488d9990179be46"
dependencies = [
"cc",
"tree-sitter",
]
[[package]]
name = "tree-sitter-ql"
version = "0.22.5"
source = "git+https://github.com/tree-sitter/tree-sitter-ql#42becd6f8f7bae82c818fa3abb1b6ff34b552310"
dependencies = [
"cc",
"tree-sitter",
] ]
[[package]] [[package]]
@@ -1900,6 +2267,12 @@ version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
[[package]]
name = "valuable"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d"
[[package]] [[package]]
name = "version_check" name = "version_check"
version = "0.9.5" version = "0.9.5"
@@ -1977,6 +2350,22 @@ version = "0.2.93"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484"
[[package]]
name = "winapi"
version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
dependencies = [
"winapi-i686-pc-windows-gnu",
"winapi-x86_64-pc-windows-gnu",
]
[[package]]
name = "winapi-i686-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
[[package]] [[package]]
name = "winapi-util" name = "winapi-util"
version = "0.1.9" version = "0.1.9"
@@ -1986,6 +2375,12 @@ dependencies = [
"windows-sys 0.59.0", "windows-sys 0.59.0",
] ]
[[package]]
name = "winapi-x86_64-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]] [[package]]
name = "windows-core" name = "windows-core"
version = "0.52.0" version = "0.52.0"
@@ -2148,3 +2543,24 @@ name = "yansi"
version = "1.0.1" version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049"
[[package]]
name = "zerocopy"
version = "0.7.35"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0"
dependencies = [
"byteorder",
"zerocopy-derive",
]
[[package]]
name = "zerocopy-derive"
version = "0.7.35"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e"
dependencies = [
"proc-macro2",
"quote",
"syn",
]

6
rust/extractor/Cargo.toml
View File

@@ -1,3 +1,5 @@
[workspace]
[package] [package]
name = "codeql-rust" name = "codeql-rust"
version = "0.1.0" version = "0.1.0"
@@ -21,6 +23,10 @@ serde = "1.0.209"
serde_with = "3.9.0" serde_with = "3.9.0"
stderrlog = "0.6.0" stderrlog = "0.6.0"
triomphe = "0.1.13" triomphe = "0.1.13"
# Ideally, we'd like to pull this in via a relative path.
# However, our bazel/rust tooling chokes on this, c.f. https://github.com/bazelbuild/rules_rust/issues/1525
# Therefore, we have a pretty bad hack in place instead, see README.md in the codeql-extractor-fake-crate directory.
codeql-extractor = { path = "codeql-extractor-fake-crate" }
[patch.crates-io] [patch.crates-io]
# patch for build script bug preventing bazel build # patch for build script bug preventing bazel build

0
rust/extractor/codeql-extractor-fake-crate/BUILD.bazel Normal file
View File

28
rust/extractor/codeql-extractor-fake-crate/Cargo.toml Normal file
View File

@@ -0,0 +1,28 @@
[package]
name = "codeql-extractor"
version = "0.2.0"
edition = "2021"
authors = ["GitHub"]
[dependencies]
flate2 = "1.0"
globset = "0.4"
tree-sitter = ">= 0.22.6"
tracing = "0.1"
tracing-subscriber = { version = "0.3.3", features = ["env-filter"] }
rayon = "1.5.0"
regex = "1.7.1"
encoding = "0.2"
lazy_static = "1.4.0"
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
chrono = { version = "0.4.19", features = ["serde"] }
num_cpus = "1.14.0"
[dev-dependencies]
tree-sitter-ql = { git = "https://github.com/tree-sitter/tree-sitter-ql" }
tree-sitter-json = {git = "https://github.com/tree-sitter/tree-sitter-json" }
rand = "0.8.5"
[patch.crates-io]
tree-sitter = {git = "https://github.com/redsun82/tree-sitter.git", rev = "1f5c1112ceaa8fc6aff61d1852690407670d2a96"}

7
rust/extractor/codeql-extractor-fake-crate/README.md Normal file
View File

@@ -0,0 +1,7 @@
We're presenting a fake crate in this workspace that ensures that the correct crate dependencies from the shared tree sitter
extractor can be parsed by Bazel (which doesn't resolve path dependencies outside of the cargo workspace unfortunately).
The sync-identical-files script keeps this up-to-date.
For local development and IDEs, we override the path to `codeql-extractor` using the `.cargo/config.toml` mechanism.
Bazel doesn't actually do anything with path dependencies except to pull in their dependency tree, so we manually
specify the dependency from the ruby extractor to the shared extractor in `BUILD.bazel`.

0
rust/extractor/codeql-extractor-fake-crate/src/lib.rs Normal file
View File

26
rust/extractor/src/config.rs
View File

@@ -1,9 +1,28 @@
use std::path::PathBuf; use std::path::PathBuf;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize, Serializer, Deserializer};
use serde_with; use serde_with;
use figment::{Figment, providers::{Env, Serialized}}; use figment::{Figment, providers::{Env, Serialized}};
use clap::{Parser, ArgAction}; use clap::{Parser, ArgAction, ValueEnum};
use clap::builder::PossibleValue;
use codeql_extractor::trap;
#[derive(Debug, PartialEq, Eq, Default, Serialize, Deserialize, Clone, Copy, ValueEnum)]
#[serde(rename_all = "lowercase")]
#[clap(rename_all = "lowercase")]
pub enum Compression {
#[default] // TODO make gzip default
None,
Gzip,
}
impl Into<trap::Compression> for Compression {
fn into(self) -> trap::Compression {
match self {
Compression::None => trap::Compression::None,
Compression::Gzip => trap::Compression::Gzip,
}
}
}
#[serde_with::apply(_ => #[serde(default)])] #[serde_with::apply(_ => #[serde(default)])]
#[derive(Debug, Deserialize, Default)] #[derive(Debug, Deserialize, Default)]
@@ -12,6 +31,7 @@ pub struct Config {
pub trap_dir: PathBuf, pub trap_dir: PathBuf,
pub source_archive_dir: PathBuf, pub source_archive_dir: PathBuf,
pub verbose: u8, pub verbose: u8,
pub compression: Compression,
pub inputs: Vec<PathBuf>, pub inputs: Vec<PathBuf>,
} }
@@ -25,6 +45,8 @@ struct CliArgs {
trap_dir: Option<PathBuf>, trap_dir: Option<PathBuf>,
#[arg(long)] #[arg(long)]
source_archive_dir: Option<PathBuf>, source_archive_dir: Option<PathBuf>,
#[arg(long)]
compression: Option<Compression>,
#[arg(short, long, action = ArgAction::Count)] #[arg(short, long, action = ArgAction::Count)]
pub verbose: u8, pub verbose: u8,

56
rust/extractor/src/generated/top.rs
View File

@@ -1,7 +1,7 @@
// generated by codegen // generated by codegen
use crate::trap::{TrapLabel, TrapId, TrapEntry, quoted}; use crate::trap::{TrapId, TrapEntry};
use std::io::Write; use codeql_extractor::trap;
#[derive(Debug)] #[derive(Debug)]
pub struct DbFile { pub struct DbFile {
@@ -14,21 +14,20 @@ impl TrapEntry for DbFile {
std::mem::replace(&mut self.id, TrapId::Star) std::mem::replace(&mut self.id, TrapId::Star)
} }
fn emit<W: Write>(self, id: TrapLabel, out: &mut W) -> std::io::Result<()> { fn emit(self, id: trap::Label, out: &mut trap::Writer) {
write!(out, "db_files({id})\n")?; out.add_tuple("db_files", vec![trap::Arg::Label(id)]);
write!(out, "files({id}, {})\n", quoted(&self.name))?; out.add_tuple("files", vec![trap::Arg::Label(id), self.name.into()]);
Ok(())
} }
} }
#[derive(Debug)] #[derive(Debug)]
pub struct DbLocation { pub struct DbLocation {
pub id: TrapId, pub id: TrapId,
pub file: TrapLabel, pub file: trap::Label,
pub start_line: u32, pub start_line: usize,
pub start_column: u32, pub start_column: usize,
pub end_line: u32, pub end_line: usize,
pub end_column: u32, pub end_column: usize,
} }
impl TrapEntry for DbLocation { impl TrapEntry for DbLocation {
@@ -36,17 +35,16 @@ impl TrapEntry for DbLocation {
std::mem::replace(&mut self.id, TrapId::Star) std::mem::replace(&mut self.id, TrapId::Star)
} }
fn emit<W: Write>(self, id: TrapLabel, out: &mut W) -> std::io::Result<()> { fn emit(self, id: trap::Label, out: &mut trap::Writer) {
write!(out, "db_locations({id})\n")?; out.add_tuple("db_locations", vec![trap::Arg::Label(id)]);
write!(out, "locations({id}, {}, {}, {}, {}, {})\n", self.file, self.start_line, self.start_column, self.end_line, self.end_column)?; out.add_tuple("locations", vec![trap::Arg::Label(id), self.file.into(), self.start_line.into(), self.start_column.into(), self.end_line.into(), self.end_column.into()]);
Ok(())
} }
} }
#[derive(Debug)] #[derive(Debug)]
pub struct Function { pub struct Function {
pub id: TrapId, pub id: TrapId,
pub location: Option<TrapLabel>, pub location: Option<trap::Label>,
pub name: String, pub name: String,
} }
@@ -55,20 +53,19 @@ impl TrapEntry for Function {
std::mem::replace(&mut self.id, TrapId::Star) std::mem::replace(&mut self.id, TrapId::Star)
} }
fn emit<W: Write>(self, id: TrapLabel, out: &mut W) -> std::io::Result<()> { fn emit(self, id: trap::Label, out: &mut trap::Writer) {
write!(out, "functions({id}, {})\n", quoted(&self.name))?; out.add_tuple("functions", vec![trap::Arg::Label(id), self.name.into()]);
if let Some(ref v) = &self.location { if let Some(v) = self.location {
write!(out, "locatable_locations({id}, {})\n", v)?; out.add_tuple("locatable_locations", vec![trap::Arg::Label(id), v.into()]);
} }
Ok(())
} }
} }
#[derive(Debug)] #[derive(Debug)]
pub struct Module { pub struct Module {
pub id: TrapId, pub id: TrapId,
pub location: Option<TrapLabel>, pub location: Option<trap::Label>,
pub declarations: Vec<TrapLabel>, pub declarations: Vec<trap::Label>,
} }
impl TrapEntry for Module { impl TrapEntry for Module {
@@ -76,14 +73,13 @@ impl TrapEntry for Module {
std::mem::replace(&mut self.id, TrapId::Star) std::mem::replace(&mut self.id, TrapId::Star)
} }
fn emit<W: Write>(self, id: TrapLabel, out: &mut W) -> std::io::Result<()> { fn emit(self, id: trap::Label, out: &mut trap::Writer) {
write!(out, "modules({id})\n")?; out.add_tuple("modules", vec![trap::Arg::Label(id)]);
if let Some(ref v) = &self.location { if let Some(v) = self.location {
write!(out, "locatable_locations({id}, {})\n", v)?; out.add_tuple("locatable_locations", vec![trap::Arg::Label(id), v.into()]);
} }
for (i, &ref v) in self.declarations.iter().enumerate() { for (i, &v) in self.declarations.iter().enumerate() {
write!(out, "module_declarations({id}, {}, {})\n", i, v)?; out.add_tuple("module_declarations", vec![trap::Arg::Label(id), i.into(), v.into()]);
} }
Ok(())
} }
} }

13
rust/extractor/src/main.rs
View File

@@ -35,21 +35,14 @@ fn main() -> anyhow::Result<()> {
prefill_caches: false, prefill_caches: false,
}; };
for input in cfg.inputs { for input in cfg.inputs {
let path = fs::canonicalize(input)?; load_workspace_at(&input, &config, &load_config, &no_progress)?;
{ let (db, vfs, _macro_server) = load_workspace_at(&input, &config, &load_config, &no_progress)?;
let mut trap = traps.create("input", &path)?;
let name = String::from(path.to_string_lossy());
trap.emit(generated::DbFile { id: name.clone().into(), name })?;
archiver.archive(&path);
}
load_workspace_at(&path, &config, &load_config, &no_progress)?;
let (db, vfs, _macro_server) = load_workspace_at(&path, &config, &load_config, &no_progress)?;
let crates = <dyn DefDatabase>::crate_graph(&db); let crates = <dyn DefDatabase>::crate_graph(&db);
for crate_id in crates.iter().take(1) { for crate_id in crates.iter().take(1) {
let krate = Crate::from(crate_id); let krate = Crate::from(crate_id);
let name = krate.display_name(&db); let name = krate.display_name(&db);
let crate_name = name.as_ref().map(|n| n.canonical_name().as_str()).unwrap_or(""); let crate_name = name.as_ref().map(|n| n.canonical_name().as_str()).unwrap_or("");
let trap = traps.create("crates", &PathBuf::from(format!("/{}_{}", crate_name, crate_id.into_raw().into_u32())))?; let trap = traps.create("crates", &PathBuf::from(format!("/{}_{}", crate_name, crate_id.into_raw().into_u32())));
translate::CrateTranslator::new( translate::CrateTranslator::new(
&db, &db,
trap, trap,

78
rust/extractor/src/translate.rs
View File

@@ -1,11 +1,10 @@
use std::collections::HashMap; use std::collections::HashMap;
use std::fs; use std::fs;
use std::path::{PathBuf}; use std::path::{PathBuf};
use crate::trap::{TrapFile, TrapId, TrapLabel}; use crate::trap::{TrapFile, TrapId, AsTrapKeyPart};
use crate::{generated, trap_key}; use crate::{generated, trap_key};
use ra_ap_hir::{Crate, Module, ModuleDef}; use ra_ap_hir::{Crate, Module, ModuleDef};
use anyhow; use anyhow;
use ra_ap_base_db::{CrateId};
use ra_ap_hir::{HasSource}; use ra_ap_hir::{HasSource};
use ra_ap_vfs::{AbsPath, FileId, Vfs}; use ra_ap_vfs::{AbsPath, FileId, Vfs};
use ra_ap_syntax::ast::HasName; use ra_ap_syntax::ast::HasName;
@@ -15,10 +14,11 @@ use triomphe::Arc;
use ra_ap_ide_db::{LineIndexDatabase, RootDatabase}; use ra_ap_ide_db::{LineIndexDatabase, RootDatabase};
use ra_ap_ide_db::line_index::LineIndex; use ra_ap_ide_db::line_index::LineIndex;
use ra_ap_syntax::AstNode; use ra_ap_syntax::AstNode;
use codeql_extractor::trap;
#[derive(Clone)] #[derive(Clone)]
struct FileData { struct FileData {
label: TrapLabel, label: trap::Label,
line_index: Arc<LineIndex>, line_index: Arc<LineIndex>,
} }
pub struct CrateTranslator<'a> { pub struct CrateTranslator<'a> {
@@ -49,55 +49,47 @@ impl CrateTranslator<'_> {
} }
} }
fn emit_file(&mut self, file_id: FileId) -> Result<Option<FileData>> { fn emit_file(&mut self, file_id: FileId) -> Option<FileData> {
if let Some(abs_path) = self.vfs.file_path(file_id).as_path() { self.vfs.file_path(file_id).as_path().and_then(|abs_path| {
let mut canonical = PathBuf::from(abs_path.as_str()); let mut canonical = PathBuf::from(abs_path.as_str());
if !self.file_labels.contains_key(&canonical) { if !self.file_labels.contains_key(&canonical) {
self.archiver.archive(&canonical); self.archiver.archive(&canonical);
canonical = fs::canonicalize(&canonical).unwrap_or(canonical); canonical = fs::canonicalize(&canonical).unwrap_or(canonical);
let name = canonical.to_string_lossy(); let name = canonical.to_string_lossy();
let label = self.trap.emit(generated::DbFile { id: trap_key!["DbFile@", name.as_ref()], name: String::from(name) })?; let label = self.trap.emit(generated::DbFile { id: trap_key!["file;", name.as_ref()], name: String::from(name) });
let line_index = <dyn LineIndexDatabase>::line_index(self.db, file_id); let line_index = <dyn LineIndexDatabase>::line_index(self.db, file_id);
self.file_labels.insert(canonical.clone(), FileData { label, line_index }); self.file_labels.insert(canonical.clone(), FileData { label, line_index });
} }
Ok(self.file_labels.get(&canonical).cloned()) self.file_labels.get(&canonical).cloned()
} else { })
Ok(None)
}
} }
fn emit_location<T: HasSource>(&mut self, entity: T) -> Result<Option<TrapLabel>> where T::Ast: AstNode { fn emit_location<T: HasSource>(&mut self, entity: T) -> Option<trap::Label>
if let Some(source) = entity.source(self.db) { where
if let Some(file_id) = source.file_id.file_id().map(|f| f.file_id()) { T::Ast: AstNode,
if let Some(data) = self.emit_file(file_id)? { {
let range = source.value.syntax().text_range(); entity.source(self.db)
let start = data.line_index.line_col(range.start()); .and_then(|source| source.file_id.file_id().map(|f| (f.file_id(), source)))
let end = data.line_index.line_col(range.end()); .and_then(|(file_id, source)| self.emit_file(file_id).map(|data| (data, source)))
return Ok(Some(self.trap.emit(generated::DbLocation { .and_then(|(data, source)| {
id: trap_key![data.label, ":", start.line, ":", start.col, ":", end.line, ":", end.col], let range = source.value.syntax().text_range();
file: data.label, let start = data.line_index.line_col(range.start());
start_line: start.line, let end = data.line_index.line_col(range.end());
start_column: start.col, Some(self.trap.emit_location(data.label, start, end))
end_line: end.line, })
end_column: end.col,
})?));
}
}
}
Ok(None)
} }
fn emit_definition(&mut self, module_label: TrapLabel, id: ModuleDef, labels: &mut Vec<TrapLabel>) -> Result<()> { fn emit_definition(&mut self, module_label: trap::Label, id: ModuleDef, labels: &mut Vec<trap::Label>) {
match id { match id {
ModuleDef::Module(_) => {} ModuleDef::Module(_) => {}
ModuleDef::Function(function) => { ModuleDef::Function(function) => {
let name = function.name(self.db); let name = function.name(self.db);
let location = self.emit_location(function)?; let location = self.emit_location(function);
labels.push(self.trap.emit(generated::Function { labels.push(self.trap.emit(generated::Function {
id: trap_key![module_label, name.as_str()], id: trap_key![module_label, name.as_str()],
location, location,
name: name.as_str().into(), name: name.as_str().into(),
})?); }));
} }
ModuleDef::Adt(_) => {} ModuleDef::Adt(_) => {}
ModuleDef::Variant(_) => {} ModuleDef::Variant(_) => {}
@@ -109,25 +101,23 @@ impl CrateTranslator<'_> {
ModuleDef::BuiltinType(_) => {} ModuleDef::BuiltinType(_) => {}
ModuleDef::Macro(_) => {} ModuleDef::Macro(_) => {}
} }
Ok(())
} }
fn emit_module(&mut self, label: TrapLabel, module: Module) -> Result<()> { fn emit_module(&mut self, label: trap::Label, module: Module) {
let mut children = Vec::new(); let mut children = Vec::new();
for id in module.declarations(self.db) { for id in module.declarations(self.db) {
self.emit_definition(label, id, &mut children)?; self.emit_definition(label, id, &mut children);
} }
self.trap.emit(generated::Module { self.trap.emit(generated::Module {
id: label.into(), id: label.into(),
location: None, location: None,
declarations: children, declarations: children,
})?; });
Ok(())
} }
pub fn emit_crate(&mut self) -> Result<()> { pub fn emit_crate(&mut self) -> std::io::Result<()> {
self.emit_file(self.krate.root_file(self.db))?; self.emit_file(self.krate.root_file(self.db));
let mut map = HashMap::<Module, TrapLabel>::new(); let mut map = HashMap::<Module, trap::Label>::new();
for module in self.krate.modules(self.db) { for module in self.krate.modules(self.db) {
let mut key = String::new(); let mut key = String::new();
if let Some(parent) = module.parent(self.db) { if let Some(parent) = module.parent(self.db) {
@@ -137,17 +127,17 @@ impl CrateTranslator<'_> {
} }
let def = module.definition_source(self.db); let def = module.definition_source(self.db);
if let Some(file) = def.file_id.file_id() { if let Some(file) = def.file_id.file_id() {
if let Some(data) = self.emit_file(file.file_id())? { if let Some(data) = self.emit_file(file.file_id()) {
key.push_str(&data.label.as_key_part()); key.push_str(&data.label.as_key_part());
} }
} }
if let Some(name) = module.name(self.db) { if let Some(name) = module.name(self.db) {
key.push_str(name.as_str()); key.push_str(name.as_str());
} }
let label = self.trap.label(TrapId::Key(key))?; let label = self.trap.label(TrapId::Key(key));
map.insert(module, label); map.insert(module, label);
self.emit_module(label, module)?; self.emit_module(label, module);
} }
Ok(()) self.trap.commit()
} }
} }

175
rust/extractor/src/trap.rs
View File

@@ -5,35 +5,39 @@ use std::io::Write;
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use log::{debug, trace}; use log::{debug, trace};
use crate::{config, path}; use crate::{config, path};
use codeql_extractor::trap;
#[derive(Clone, Copy)] use ra_ap_ide_db::line_index::LineCol;
pub struct TrapLabel(u64); use crate::config::Compression;
use crate::generated;
impl Debug for TrapLabel {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
write!(f, "TrapLabel({:x})", self.0)
}
}
impl Display for TrapLabel {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
write!(f, "#{:x}", self.0)
}
}
//TODO: typed labels //TODO: typed labels
pub trait AsTrapKeyPart {
fn as_key_part(&self) -> String;
}
impl TrapLabel { impl AsTrapKeyPart for trap::Label {
pub fn as_key_part(&self) -> String { fn as_key_part(&self) -> String {
format!("{{{}}}", self) format!("{{{}}}", self)
} }
} }
impl AsTrapKeyPart for String {
fn as_key_part(&self) -> String {
self.clone()
}
}
impl AsTrapKeyPart for &str {
fn as_key_part(&self) -> String {
String::from(*self)
}
}
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub enum TrapId { pub enum TrapId {
Star, Star,
Key(String), Key(String),
Label(TrapLabel), Label(trap::Label),
} }
impl From<String> for TrapId { impl From<String> for TrapId {
@@ -48,8 +52,8 @@ impl From<&str> for TrapId {
} }
} }
impl From<TrapLabel> for TrapId { impl From<trap::Label> for TrapId {
fn from(value: TrapLabel) -> Self { fn from(value: trap::Label) -> Self {
TrapId::Label(value) TrapId::Label(value)
} }
} }
@@ -57,12 +61,6 @@ impl From<TrapLabel> for TrapId {
#[macro_export] #[macro_export]
macro_rules! trap_key { macro_rules! trap_key {
($($x:expr),+ $(,)?) => {{ ($($x:expr),+ $(,)?) => {{
trait BlanketKeyPart: std::fmt::Display {
fn as_key_part(&self) -> String {
format!("{}", self)
}
}
impl<T: std::fmt::Display> BlanketKeyPart for T {}
let mut key = String::new(); let mut key = String::new();
$( $(
key.push_str(&$x.as_key_part()); key.push_str(&$x.as_key_part());
@@ -71,80 +69,63 @@ macro_rules! trap_key {
}}; }};
} }
impl Display for TrapId {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
match self {
TrapId::Star => write!(f, "*"),
TrapId::Key(k) => write!(f, "@{}", quoted(k)),
TrapId::Label(l) => Display::fmt(&l, f)
}
}
}
pub fn escaped(s: &str) -> String {
s.replace("\"", "\"\"")
}
pub fn quoted(s: &str) -> String {
format!("\"{}\"", escaped(s))
}
pub trait TrapEntry: std::fmt::Debug { pub trait TrapEntry: std::fmt::Debug {
fn extract_id(&mut self) -> TrapId; fn extract_id(&mut self) -> TrapId;
fn emit<W: Write>(self, id: TrapLabel, out: &mut W) -> std::io::Result<()>; fn emit(self, id: trap::Label, out: &mut trap::Writer);
} }
#[derive(Debug)]
pub struct TrapFile { pub struct TrapFile {
label_index: u64, path: PathBuf,
file: File, writer: trap::Writer,
trap_name: String, compression: Compression,
} }
impl TrapFile { impl TrapFile {
pub fn comment(&mut self, message: &str) -> std::io::Result<()> { pub fn emit_location(&mut self, file_label: trap::Label, start: LineCol, end: LineCol) -> trap::Label {
for part in message.split("\n") { let start_line = start.line as usize;
trace!("emit -> {}: // {part}", self.trap_name); let start_column = start.col as usize;
write!(self.file, "// {part}\n")?; let end_line = end.line as usize;
} let end_column = end.col as usize;
Ok(()) let (ret, _) = self.writer.location_label(trap::Location {
} start_line,
start_column,
pub fn label(&mut self, mut id: TrapId) -> std::io::Result<TrapLabel> { end_line,
match id { end_column,
TrapId::Star => {} });
TrapId::Key(ref s) => { self.emit(generated::DbLocation {
if s.is_empty() { id: ret.into(),
id = TrapId::Star; file: file_label,
} start_line,
} start_column,
TrapId::Label(l) => { end_line,
return Ok(l); end_column,
} });
}
let ret = self.create_label();
trace!("emit -> {}: {ret:?} = {id:?}", self.trap_name);
write!(self.file, "{ret}={id}\n")?;
Ok(ret)
}
pub fn emit<T: TrapEntry>(&mut self, mut entry: T) -> std::io::Result<TrapLabel> {
trace!("emit -> {}: {entry:?}", self.trap_name);
let id = self.label(entry.extract_id())?;
entry.emit(id, &mut self.file)?;
Ok(id)
}
fn create_label(&mut self) -> TrapLabel {
let ret = TrapLabel(self.label_index);
self.label_index += 1;
ret ret
} }
pub fn label(&mut self, id: TrapId) -> trap::Label {
match id {
TrapId::Star => self.writer.fresh_id(),
TrapId::Key(s) => self.writer.global_id(&s).0,
TrapId::Label(l) => l,
}
}
pub fn emit<T: TrapEntry>(&mut self, mut e: T) -> trap::Label {
let label = self.label(e.extract_id());
e.emit(label, &mut self.writer);
label
}
pub fn commit(&self) -> std::io::Result<()> {
std::fs::create_dir_all(self.path.parent().unwrap())?;
self.writer.write_to_file(&self.path, self.compression.into())
}
} }
pub struct TrapFileProvider { pub struct TrapFileProvider {
trap_dir: PathBuf, trap_dir: PathBuf,
compression: Compression,
} }
impl TrapFileProvider { impl TrapFileProvider {
@@ -152,27 +133,25 @@ impl TrapFileProvider {
let trap_dir = cfg.trap_dir.clone(); let trap_dir = cfg.trap_dir.clone();
std::fs::create_dir_all(&trap_dir)?; std::fs::create_dir_all(&trap_dir)?;
Ok(TrapFileProvider { Ok(TrapFileProvider {
trap_dir trap_dir,
compression: cfg.compression,
}) })
} }
pub fn create(&self, category: &str, key: &Path) -> std::io::Result<TrapFile> { pub fn create(&self, category: &str, key: &Path) -> TrapFile {
let mut path = PathBuf::from(category); let mut path = PathBuf::from(category);
path.push(path::key(key)); path.push(path::key(key));
path.set_extension(path.extension().map(|e| { path.set_extension(path.extension().map(|e| {
let mut o : OsString = e.to_owned(); let mut o: OsString = e.to_owned();
o.push(".trap"); o.push(".trap");
o o
}).unwrap_or("trap".into())); }).unwrap_or("trap".into()));
let trap_name = String::from(path.to_string_lossy()); debug!("creating trap file {}", path.display());
debug!("creating trap file {}", trap_name);
path = self.trap_dir.join(path); path = self.trap_dir.join(path);
std::fs::create_dir_all(path.parent().unwrap())?; TrapFile {
let file = File::create(path)?; path,
Ok(TrapFile { writer: trap::Writer::new(),
label_index: 0, compression: self.compression,
file, }
trap_name,
})
} }
} }

32
shared/tree-sitter-extractor/src/trap.rs
View File

@@ -136,10 +136,16 @@ impl fmt::Display for Entry {
} }
} }
#[derive(Debug, Copy, Clone)] #[derive(Copy, Clone)]
// Identifiers of the form #0, #1... // Identifiers of the form #0, #1...
pub struct Label(u32); pub struct Label(u32);
impl fmt::Debug for Label {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> fmt::Result {
write!(f, "Label({:#x})", self.0)
}
}
impl fmt::Display for Label { impl fmt::Display for Label {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "#{:x}", self.0) write!(f, "#{:x}", self.0)
@@ -170,6 +176,30 @@ impl fmt::Display for Arg {
} }
} }
impl From<String> for Arg {
fn from(value: String) -> Self {
Arg::String(value)
}
}
impl From<&str> for Arg {
fn from(value: &str) -> Self {
Arg::String(value.into())
}
}
impl From<Label> for Arg {
fn from(value: Label) -> Self {
Arg::Label(value)
}
}
impl From<usize> for Arg {
fn from(value: usize) -> Self {
Arg::Int(value)
}
}
pub struct Program(Vec<Entry>); pub struct Program(Vec<Entry>);
impl fmt::Display for Program { impl fmt::Display for Program {