hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2026-04-27 17:55:19 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into rust-taint

Browse Source
This commit is contained in:
Simon Friis Vindum
2024-12-04 15:28:30 +01:00
parent 3004639fca f56b2c912a
commit d6ab7d2aff
243 changed files with 1651 additions and 1061 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
rust/extractor/Cargo.toml
View File

@@ -33,4 +33,6 @@ codeql-extractor = { path = "../../shared/tree-sitter-extractor" }
rust-extractor-macros = { path = "macros" }
itertools = "0.13.0"
glob = "0.3.1"
chrono = { version = "0.4.38", features = ["serde"] }
serde_json = "1.0.133"
dunce = "1.0.5"

1
rust/extractor/src/config.rs
View File

@@ -45,6 +45,7 @@ pub struct Config {
pub scratch_dir: PathBuf,
pub trap_dir: PathBuf,
pub source_archive_dir: PathBuf,
pub diagnostic_dir: PathBuf,
pub cargo_target_dir: Option<PathBuf>,
pub cargo_target: Option<String>,
pub cargo_features: Vec<String>,

255
rust/extractor/src/diagnostics.rs Normal file
View File

@@ -0,0 +1,255 @@
use crate::config::Config;
use anyhow::Context;
use chrono::{DateTime, Utc};
use log::{debug, info};
use ra_ap_project_model::ProjectManifest;
use serde::ser::SerializeMap;
use serde::Serialize;
use std::collections::HashMap;
use std::fmt::Display;
use std::fs::File;
use std::path::{Path, PathBuf};
use std::time::Instant;
#[derive(Default, Debug, Clone, Copy, Serialize)]
#[serde(rename_all = "camelCase")]
#[allow(dead_code)]
enum Severity {
#[default]
Note,
Warning,
Error,
}
#[derive(Default, Debug, Clone, Copy, Serialize)]
#[serde(rename_all = "camelCase")]
struct Visibility {
status_page: bool,
cli_summary_table: bool,
telemetry: bool,
}
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
#[allow(dead_code)]
enum Message {
TextMessage(String),
MarkdownMessage(String),
}
impl Default for Message {
fn default() -> Self {
Message::TextMessage("".to_string())
}
}
#[derive(Default, Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
struct Source {
id: String,
name: String,
extractor_name: String,
}
#[derive(Default, Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
struct Location {
file: PathBuf,
start_line: u32,
start_column: u32,
end_line: u32,
end_column: u32,
}
#[derive(Default, Debug, Clone, Serialize)]
pub struct Diagnostics<T> {
source: Source,
visibility: Visibility,
severity: Severity,
#[serde(flatten)]
message: Message,
timestamp: DateTime<Utc>,
#[serde(skip_serializing_if = "Option::is_none")]
location: Option<Location>,
attributes: T,
}
#[derive(Default, Debug, Clone, Copy, Serialize, PartialEq, Eq, Hash)]
#[serde(rename_all = "camelCase")]
pub enum ExtractionStepKind {
#[default]
LoadManifest,
LoadSource,
Parse,
Extract,
}
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct ExtractionStep {
pub action: ExtractionStepKind,
pub file: PathBuf,
pub ms: u128,
}
impl ExtractionStep {
fn new(start: Instant, action: ExtractionStepKind, file: PathBuf) -> Self {
let ret = ExtractionStep {
action,
file,
ms: start.elapsed().as_millis(),
};
debug!("{ret:?}");
ret
}
pub fn load_manifest(start: Instant, target: &ProjectManifest) -> Self {
Self::new(
start,
ExtractionStepKind::LoadManifest,
PathBuf::from(target.manifest_path()),
)
}
pub fn parse(start: Instant, target: &Path) -> Self {
Self::new(start, ExtractionStepKind::Parse, PathBuf::from(target))
}
pub fn extract(start: Instant, target: &Path) -> Self {
Self::new(start, ExtractionStepKind::Extract, PathBuf::from(target))
}
pub fn load_source(start: Instant, target: &Path) -> Self {
Self::new(start, ExtractionStepKind::LoadSource, PathBuf::from(target))
}
}
#[derive(Debug, Default, Clone)]
struct HumanReadableDuration(u128);
impl Serialize for HumanReadableDuration {
fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
let mut map = serializer.serialize_map(Some(2))?;
map.serialize_entry("ms", &self.0)?;
map.serialize_entry("pretty", &self.pretty())?;
map.end()
}
}
impl HumanReadableDuration {
pub fn add(&mut self, other: u128) {
self.0 += other;
}
pub fn pretty(&self) -> String {
let milliseconds = self.0 % 1000;
let mut seconds = self.0 / 1000;
if seconds < 60 {
return format!("{seconds}.{milliseconds:03}s");
}
let mut minutes = seconds / 60;
seconds %= 60;
if minutes < 60 {
return format!("{minutes}min{seconds:02}.{milliseconds:03}s");
}
let hours = minutes / 60;
minutes %= 60;
format!("{hours}h{minutes:02}min{seconds:02}.{milliseconds:03}s")
}
}
impl From<u128> for HumanReadableDuration {
fn from(val: u128) -> Self {
HumanReadableDuration(val)
}
}
impl Display for HumanReadableDuration {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
f.write_str(&self.pretty())
}
}
#[derive(Debug, Default, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
struct DurationsSummary {
#[serde(flatten)]
durations: HashMap<ExtractionStepKind, HumanReadableDuration>,
total: HumanReadableDuration,
}
#[derive(Debug, Default, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
struct ExtractionSummary {
number_of_manifests: usize,
number_of_files: usize,
durations: DurationsSummary,
}
type ExtractionDiagnostics = Diagnostics<ExtractionSummary>;
fn summary(start: Instant, steps: &[ExtractionStep]) -> ExtractionSummary {
let mut number_of_manifests = 0;
let mut number_of_files = 0;
let mut durations = HashMap::new();
for step in steps {
match &step.action {
ExtractionStepKind::LoadManifest => {
number_of_manifests += 1;
}
ExtractionStepKind::Parse => {
number_of_files += 1;
}
_ => {}
}
durations
.entry(step.action)
.or_insert(HumanReadableDuration(0))
.add(step.ms);
}
let total = start.elapsed().as_millis().into();
for (key, value) in &durations {
info!("total duration ({key:?}): {value}");
}
info!("total duration: {total}");
ExtractionSummary {
number_of_manifests,
number_of_files,
durations: DurationsSummary { durations, total },
}
}
pub fn emit_extraction_diagnostics(
start: Instant,
config: &Config,
steps: &[ExtractionStep],
) -> anyhow::Result<()> {
let summary = summary(start, steps);
let diagnostics = ExtractionDiagnostics {
source: Source {
id: "rust/extractor/telemetry".to_owned(),
name: "telemetry".to_string(),
extractor_name: "rust".to_string(),
},
visibility: Visibility {
telemetry: true,
..Default::default()
},
timestamp: Utc::now(),
attributes: summary,
..Default::default()
};
std::fs::create_dir_all(&config.diagnostic_dir).with_context(|| {
format!(
"creating diagnostics directory {}",
config.diagnostic_dir.display()
)
})?;
let target = config.diagnostic_dir.join("extraction.jsonc");
let mut output = File::create(&target)
.with_context(|| format!("creating diagnostics file {}", target.display()))?;
serde_json::to_writer_pretty(&mut output, &diagnostics)
.with_context(|| format!("writing to diagnostics file {}", target.display()))?;
Ok(())
}

2
rust/extractor/src/generated/.generated.list generated
View File

@@ -1,2 +1,2 @@
mod.rs 4bcb9def847469aae9d8649461546b7c21ec97cf6e63d3cf394e339915ce65d7 4bcb9def847469aae9d8649461546b7c21ec97cf6e63d3cf394e339915ce65d7
top.rs aad41fc41df08b35aeca0700426e38538c23aef9030b42aeaac964dced524575 aad41fc41df08b35aeca0700426e38538c23aef9030b42aeaac964dced524575
top.rs c77ccf1e73a5b139f768e80580a261f7f3ab76823a1c9095b06f704a4912e8f1 c77ccf1e73a5b139f768e80580a261f7f3ab76823a1c9095b06f704a4912e8f1

40
rust/extractor/src/generated/top.rs generated
View File

@@ -4,6 +4,15 @@
use crate::trap;
#[derive(Debug)]
pub struct File {
_unused: ()
}
impl trap::TrapClass for File {
fn class_name() -> &'static str { "File" }
}
#[derive(Debug)]
pub struct Element {
_unused: ()
@@ -13,6 +22,37 @@ impl trap::TrapClass for Element {
fn class_name() -> &'static str { "Element" }
}
#[derive(Debug)]
pub struct ExtractorStep {
pub id: trap::TrapId<ExtractorStep>,
pub action: String,
pub file: trap::Label<File>,
pub duration_ms: usize,
}
impl trap::TrapEntry for ExtractorStep {
fn extract_id(&mut self) -> trap::TrapId<Self> {
std::mem::replace(&mut self.id, trap::TrapId::Star)
}
fn emit(self, id: trap::Label<Self>, out: &mut trap::Writer) {
out.add_tuple("extractor_steps", vec![id.into(), self.action.into(), self.file.into(), self.duration_ms.into()]);
}
}
impl trap::TrapClass for ExtractorStep {
fn class_name() -> &'static str { "ExtractorStep" }
}
impl From<trap::Label<ExtractorStep>> for trap::Label<Element> {
fn from(value: trap::Label<ExtractorStep>) -> Self {
// SAFETY: this is safe because in the dbscheme ExtractorStep is a subclass of Element
unsafe {
Self::from_untyped(value.as_untyped())
}
}
}
#[derive(Debug)]
pub struct Locatable {
_unused: ()

109
rust/extractor/src/main.rs
View File

@@ -1,13 +1,16 @@
use crate::diagnostics::{emit_extraction_diagnostics, ExtractionStep};
use crate::rust_analyzer::path_to_file_id;
use crate::trap::TrapId;
use anyhow::Context;
use archive::Archiver;
use log::info;
use log::{info, warn};
use ra_ap_hir::Semantics;
use ra_ap_ide_db::line_index::{LineCol, LineIndex};
use ra_ap_ide_db::RootDatabase;
use ra_ap_project_model::ProjectManifest;
use ra_ap_project_model::{CargoConfig, ProjectManifest};
use ra_ap_vfs::Vfs;
use rust_analyzer::{ParseResult, RustAnalyzer};
use std::time::Instant;
use std::{
collections::HashMap,
path::{Path, PathBuf},
@@ -15,6 +18,7 @@ use std::{
mod archive;
mod config;
mod diagnostics;
pub mod generated;
mod qltest;
mod rust_analyzer;
@@ -24,18 +28,31 @@ pub mod trap;
struct Extractor<'a> {
archiver: &'a Archiver,
traps: &'a trap::TrapFileProvider,
steps: Vec<ExtractionStep>,
}
impl Extractor<'_> {
fn extract(&self, rust_analyzer: &rust_analyzer::RustAnalyzer, file: &std::path::Path) {
impl<'a> Extractor<'a> {
pub fn new(archiver: &'a Archiver, traps: &'a trap::TrapFileProvider) -> Self {
Self {
archiver,
traps,
steps: Vec::new(),
}
}
fn extract(&mut self, rust_analyzer: &rust_analyzer::RustAnalyzer, file: &std::path::Path) {
self.archiver.archive(file);
let before_parse = Instant::now();
let ParseResult {
ast,
text,
errors,
semantics_info,
} = rust_analyzer.parse(file);
self.steps.push(ExtractionStep::parse(before_parse, file));
let before_extract = Instant::now();
let line_index = LineIndex::new(text.as_ref());
let display_path = file.to_string_lossy();
let mut trap = self.traps.create("source", file);
@@ -73,22 +90,79 @@ impl Extractor<'_> {
err.to_string()
)
});
self.steps
.push(ExtractionStep::extract(before_extract, file));
}
pub fn extract_with_semantics(
&self,
&mut self,
file: &Path,
semantics: &Semantics<'_, RootDatabase>,
vfs: &Vfs,
) {
self.extract(&RustAnalyzer::new(vfs, semantics), file);
}
pub fn extract_without_semantics(&self, file: &Path, reason: &str) {
pub fn extract_without_semantics(&mut self, file: &Path, reason: &str) {
self.extract(&RustAnalyzer::WithoutSemantics { reason }, file);
}
pub fn load_manifest(
&mut self,
project: &ProjectManifest,
config: &CargoConfig,
) -> Option<(RootDatabase, Vfs)> {
let before = Instant::now();
let ret = RustAnalyzer::load_workspace(project, config);
self.steps
.push(ExtractionStep::load_manifest(before, project));
ret
}
pub fn load_source(
&mut self,
file: &Path,
semantics: &Semantics<'_, RootDatabase>,
vfs: &Vfs,
) -> Result<(), String> {
let before = Instant::now();
let Some(id) = path_to_file_id(file, vfs) else {
return Err("not included in files loaded from manifest".to_string());
};
if semantics.file_to_module_def(id).is_none() {
return Err("not included as a module".to_string());
}
self.steps.push(ExtractionStep::load_source(before, file));
Ok(())
}
pub fn emit_extraction_diagnostics(
self,
start: Instant,
cfg: &config::Config,
) -> anyhow::Result<()> {
emit_extraction_diagnostics(start, cfg, &self.steps)?;
let mut trap = self.traps.create("diagnostics", "extraction");
for step in self.steps {
let file = trap.emit_file(&step.file);
let duration_ms = usize::try_from(step.ms).unwrap_or_else(|_e| {
warn!("extraction step duration overflowed ({step:?})");
i32::MAX as usize
});
trap.emit(generated::ExtractorStep {
id: TrapId::Star,
action: format!("{:?}", step.action),
file,
duration_ms,
});
}
trap.commit()?;
Ok(())
}
}
fn main() -> anyhow::Result<()> {
let start = Instant::now();
let mut cfg = config::Config::extract().context("failed to load configuration")?;
stderrlog::new()
.module(module_path!())
@@ -103,10 +177,7 @@ fn main() -> anyhow::Result<()> {
let archiver = archive::Archiver {
root: cfg.source_archive_dir.clone(),
};
let extractor = Extractor {
archiver: &archiver,
traps: &traps,
};
let mut extractor = Extractor::new(&archiver, &traps);
let files: Vec<PathBuf> = cfg
.inputs
.iter()
@@ -132,21 +203,13 @@ fn main() -> anyhow::Result<()> {
}
let cargo_config = cfg.to_cargo_config();
for (manifest, files) in map.values().filter(|(_, files)| !files.is_empty()) {
if let Some((ref db, ref vfs)) = RustAnalyzer::load_workspace(manifest, &cargo_config) {
if let Some((ref db, ref vfs)) = extractor.load_manifest(manifest, &cargo_config) {
let semantics = Semantics::new(db);
for file in files {
let Some(id) = path_to_file_id(file, vfs) else {
extractor.extract_without_semantics(
file,
"not included in files loaded from manifest",
);
continue;
match extractor.load_source(file, &semantics, vfs) {
Ok(()) => extractor.extract_with_semantics(file, &semantics, vfs),
Err(reason) => extractor.extract_without_semantics(file, &reason),
};
if semantics.file_to_module_def(id).is_none() {
extractor.extract_without_semantics(file, "not included as a module");
continue;
}
extractor.extract_with_semantics(file, &semantics, vfs);
}
} else {
for file in files {
@@ -155,5 +218,5 @@ fn main() -> anyhow::Result<()> {
}
}
Ok(())
extractor.emit_extraction_diagnostics(start, &cfg)
}

5
rust/extractor/src/translate/base.rs
View File

@@ -4,7 +4,6 @@ use crate::generated::{self};
use crate::rust_analyzer::FileSemanticInformation;
use crate::trap::{DiagnosticSeverity, TrapFile, TrapId};
use crate::trap::{Label, TrapClass};
use codeql_extractor::trap::{self};
use itertools::Either;
use log::Level;
use ra_ap_base_db::salsa::InternKey;
@@ -65,7 +64,7 @@ macro_rules! emit_detached {
pub struct Translator<'a> {
pub trap: TrapFile,
path: &'a str,
label: trap::Label,
label: Label<generated::File>,
line_index: LineIndex,
file_id: Option<EditionedFileId>,
pub semantics: Option<&'a Semantics<'a, RootDatabase>>,
@@ -75,7 +74,7 @@ impl<'a> Translator<'a> {
pub fn new(
trap: TrapFile,
path: &'a str,
label: trap::Label,
label: Label<generated::File>,
line_index: LineIndex,
semantic_info: Option<&FileSemanticInformation<'a>>,
) -> Translator<'a> {

18
rust/extractor/src/trap.rs
View File

@@ -1,5 +1,5 @@
use crate::config;
use crate::config::Compression;
use crate::{config, generated};
use codeql_extractor::{extractor, file_paths, trap};
use log::debug;
use ra_ap_ide_db::line_index::LineCol;
@@ -138,7 +138,7 @@ pub enum DiagnosticSeverity {
impl TrapFile {
pub fn emit_location_label(
&mut self,
file_label: UntypedLabel,
file_label: Label<generated::File>,
start: LineCol,
end: LineCol,
) -> UntypedLabel {
@@ -149,7 +149,7 @@ impl TrapFile {
extractor::location_label(
&mut self.writer,
trap::Location {
file_label,
file_label: file_label.as_untyped(),
start_line,
start_column,
end_line,
@@ -159,7 +159,7 @@ impl TrapFile {
}
pub fn emit_location<E: TrapClass>(
&mut self,
file_label: UntypedLabel,
file_label: Label<generated::File>,
entity_label: Label<E>,
start: LineCol,
end: LineCol,
@@ -192,8 +192,10 @@ impl TrapFile {
],
);
}
pub fn emit_file(&mut self, absolute_path: &Path) -> trap::Label {
extractor::populate_file(&mut self.writer, absolute_path)
pub fn emit_file(&mut self, absolute_path: &Path) -> Label<generated::File> {
let untyped = extractor::populate_file(&mut self.writer, absolute_path);
// SAFETY: populate_file emits `@file` typed labels
unsafe { Label::from_untyped(untyped) }
}
pub fn label<T: TrapEntry>(&mut self, id: TrapId<T>) -> Label<T> {
@@ -243,8 +245,8 @@ impl TrapFileProvider {
})
}
pub fn create(&self, category: &str, key: &Path) -> TrapFile {
let path = file_paths::path_for(&self.trap_dir.join(category), key, "trap");
pub fn create(&self, category: &str, key: impl AsRef<Path>) -> TrapFile {
let path = file_paths::path_for(&self.trap_dir.join(category), key.as_ref(), "trap");
debug!("creating trap file {}", path.display());
let mut writer = trap::Writer::new();
extractor::populate_empty_location(&mut writer);

13
rust/ql/.generated.list generated
View File

@@ -226,6 +226,8 @@ lib/codeql/rust/elements/internal/ExternCrateImpl.qll ade4df9d3f87daf6534b8e79ff
lib/codeql/rust/elements/internal/ExternItemImpl.qll 577c8ac387c47746e3b45f943374c7ab641e8ad119e8591c31f219a5f08d3a29 bba88b974d1c03c78e0caf3d8f4118426d2aa8bd6ffd6f59a3da8ff1524a173f
lib/codeql/rust/elements/internal/ExternItemListConstructor.qll 9e4f6a036707c848c0553119272fd2b11c1740dd9910a626a9a0cf68a55b249b efde86b18bd419154fb5b6d28790a14ea989b317d84b5c1ddbdfb29c6924fd86
lib/codeql/rust/elements/internal/ExternItemListImpl.qll e89d0cf938f6e137ba1ce7907a923b1ab2be7be2fdd642c3b7a722f11b9199f8 85906d3ce89e5abc301cc96ea5104d53e90af3f5f22f8d54ec437687096e39d8
lib/codeql/rust/elements/internal/ExtractorStep.qll 1c65668007ea71d05333e44132eccc01dc2a2b4908fb37d0a73995119d3ed5f0 8cbe1eeb35bc2bc95c1b7765070d1ff58aae03fd28dc94896b091858eea40efe
lib/codeql/rust/elements/internal/ExtractorStepConstructor.qll 00c527a3139ad399ea1efd0ebe4656372d70f6c4e79136bc497a6cb84becae8e 93817f3dddeaf2c0964ab31c2df451dcee0aeba7cb6520803d8ce42cefcb3703
lib/codeql/rust/elements/internal/FieldExprConstructor.qll b3be2c4ccaf2c8a1283f3d5349d7f4f49f87b35e310ef33491023c5ab6f3abc5 645d0d4073b032f6b7284fc36a10a6ec85596fb95c68f30c09504f2c5a6f789f
lib/codeql/rust/elements/internal/FieldListImpl.qll 02a09d1d146030c68cead4614f4eef75854f19e55ed1eda60b34c4858a8d4a88 9b9f5e77546434c771d2f785119577ec46569a18473daa4169fb84a097369493
lib/codeql/rust/elements/internal/FnPtrTypeReprConstructor.qll 61d8808ea027a6e04d5304c880974332a0195451f6b4474f84b3695ec907d865 0916c63a02b01a839fe23ec8b189d37dc1b8bc4e1ba753cbf6d6f5067a46965a
@@ -455,6 +457,7 @@ lib/codeql/rust/elements/internal/generated/ExternBlock.qll c292d804a1f8d2cf6a44
lib/codeql/rust/elements/internal/generated/ExternCrate.qll 35fea4e810a896c1656adb4682c4c3bc20283768073e26ae064189ce310433c8 fc504dff79ba758d89b10cd5049539fbc766ee9862ff495066cea26abf0b5e0b
lib/codeql/rust/elements/internal/generated/ExternItem.qll 749b064ad60f32197d5b85e25929afe18e56e12f567b73e21e43e2fdf4c447e3 e2c2d423876675cf2dae399ca442aef7b2860319da9bfadeff29f2c6946f8de7
lib/codeql/rust/elements/internal/generated/ExternItemList.qll 6bc97fdae6c411cab5c501129c1d6c2321c1011cccb119515d75d07dc55c253b 6b5aa808025c0a4270cac540c07ba6faede1b3c70b8db5fd89ec5d46df9041b2
lib/codeql/rust/elements/internal/generated/ExtractorStep.qll b83ce7f18009bdd36374260652c2a8a5cd5a9b5404a1c147bbec49ad251e43f3 e6e55595300126f9c5a6fd7bde5321b2a0026b491326114d16fcc2395a1fc483
lib/codeql/rust/elements/internal/generated/FieldExpr.qll 3e506b5cb93793ec30f56bb637a600db869fcba6181b068516a671d55c362739 7bbf953696d763ad6b210f378f487ba85b875fa115b22c0c0508599a63633502
lib/codeql/rust/elements/internal/generated/FieldList.qll 43c13c6e3c9ba75a7a4cb870fc4f18752001584d48b9df0734055a6ebb789331 7c51b0b13eb02f1286d3365e53a976ba2655c4dbd8e735bc11c8b205c829e1ee
lib/codeql/rust/elements/internal/generated/FnPtrTypeRepr.qll d490ab9f2e3654d9abde18a06e534abd99ca62f518ca08670b696a97e9d5c592 01500319820f66cb4bbda6fe7c26270f76ea934efff4bb3cbf88e9b1e07e8be2
@@ -518,7 +521,7 @@ lib/codeql/rust/elements/internal/generated/ParamList.qll c808c9d84dd7800573832b
lib/codeql/rust/elements/internal/generated/ParenExpr.qll bc0731505bfe88516205ec360582a4222d2681d11342c93e15258590ddee82f2 d4bd6e0c80cf1d63746c88d4bcb3a01d4c75732e5da09e3ebd9437ced227fb60
lib/codeql/rust/elements/internal/generated/ParenPat.qll 4f168ef5d5bb87a903251cc31b2e44a759b099ec69c90af31783fbb15778c940 0e34f94a45a13396fd57d94c245dc64d1adde2ab0e22b56946f7e94c04e297fc
lib/codeql/rust/elements/internal/generated/ParenTypeRepr.qll 40ab5c592e7699c621787793743e33988de71ff42ca27599f5ab3ddb70e3f7d8 12c0a6eed2202ee3e892f61da3b3ce77ac3190854cdf3097e8d2be98aa3cb91d
lib/codeql/rust/elements/internal/generated/ParentChild.qll c3f6352ef56aabb6bc784deafc35807b32df6f41c3bca3437221166776885c2d 3f04580798ab351e5835957379ced2b45fec628b76578c0f33f571cf03abfc83
lib/codeql/rust/elements/internal/generated/ParentChild.qll ed0af2cad8ec4d612e3c03d99548444298ae975516f2c3e909026fe5d31ab467 57801012cbe803516a093bbc0c5dcf839cb43df97a87fd2028ad2be400882c50
lib/codeql/rust/elements/internal/generated/Pat.qll 3605ac062be2f294ee73336e9669027b8b655f4ad55660e1eab35266275154ee 7f9400db2884d336dd1d21df2a8093759c2a110be9bf6482ce8e80ae0fd74ed4
lib/codeql/rust/elements/internal/generated/Path.qll 4c1c8e840ed57880e574142b081b11d7a7428a009f10e3aa8f4645e211f6b2e0 989668cf0f1bdee7557e2f97c01e41d2a56848227fed41477833f5fc1e1d35f6
lib/codeql/rust/elements/internal/generated/PathExpr.qll 4ff4b2ab77bce1dbfddb315e7d1ff13d6fcd6bb7c30c105402f8082d05f1d337 fbc4f4e05da75ab543af33cfb620cfe09239e2574b8312f2c5bedca8a65ea6e8
@@ -531,7 +534,7 @@ lib/codeql/rust/elements/internal/generated/PtrTypeRepr.qll 51d1e9e683fc79dddbff
lib/codeql/rust/elements/internal/generated/PureSynthConstructors.qll e5b8e69519012bbaae29dcb82d53f7f7ecce368c0358ec27ef6180b228a0057f e5b8e69519012bbaae29dcb82d53f7f7ecce368c0358ec27ef6180b228a0057f
lib/codeql/rust/elements/internal/generated/RangeExpr.qll 23cca03bf43535f33b22a38894f70d669787be4e4f5b8fe5c8f7b964d30e9027 18624cef6c6b679eeace2a98737e472432e0ead354cca02192b4d45330f047c9
lib/codeql/rust/elements/internal/generated/RangePat.qll 80826a6a6868a803aa2372e31c52a03e1811a3f1f2abdb469f91ca0bfdd9ecb6 34ee1e208c1690cba505dff2c588837c0cd91e185e2a87d1fe673191962276a9
lib/codeql/rust/elements/internal/generated/Raw.qll 7ffb00a545dfe16556b60a92f118c1175544f07ece90b7f46db2c119a2481753 e6fd9bb3da185bcfbb55f477f0ef31f689df7d66b76bcf93e29e020a67f07f42
lib/codeql/rust/elements/internal/generated/Raw.qll e2dc96e3d33779e31424809f1048d2f1010280af20dd6f89e3939512009ba3a7 925887ef2978a0b8351ff0046ae0bc410c5c4b7764a4aa4aa2b75f6ba3098710
lib/codeql/rust/elements/internal/generated/RecordExpr.qll eb6cb662e463f9260efae1a6ce874fa781172063b916ef1963f861e9942d308d 1a21cbccc8f3799ff13281e822818ebfb21d81591720a427cac3625512cb9d40
lib/codeql/rust/elements/internal/generated/RecordExprField.qll 7e9f8663d3b74ebbc9603b10c9912f082febba6bd73d344b100bbd3edf837802 fbe6b578e7fd5d5a6f21bbb8c388957ab7210a6a249ec71510a50fb35b319ea1
lib/codeql/rust/elements/internal/generated/RecordExprFieldList.qll 179a97211fe7aa6265085d4d54115cdbc0e1cd7c9b2135591e8f36d6432f13d3 dd44bbbc1e83a1ed3a587afb729d7debf7aeb7b63245de181726af13090e50c0
@@ -557,8 +560,8 @@ lib/codeql/rust/elements/internal/generated/Static.qll ea22838e0b7d9796dfaf5deda
lib/codeql/rust/elements/internal/generated/Stmt.qll 8473ff532dd5cc9d7decaddcd174b94d610f6ca0aec8e473cc051dad9f3db917 6ef7d2b5237c2dbdcacbf7d8b39109d4dc100229f2b28b5c9e3e4fbf673ba72b
lib/codeql/rust/elements/internal/generated/StmtList.qll a667193e32341e17400867c6e359878c4e645ef9f5f4d97676afc0283a33a026 a320ed678ee359302e2fc1b70a9476705cd616fcfa44a499d32f0c7715627f73
lib/codeql/rust/elements/internal/generated/Struct.qll 4d57f0db12dc7ad3e31e750a24172ef1505406b4dab16386af0674bd18bf8f4b 1a73c83df926b996f629316f74c61ea775be04532ab61b56af904223354f033e
lib/codeql/rust/elements/internal/generated/Synth.qll 0bb53e7d628fcd74a28fb7b00c51dc7f212a775894a4dc485a68089a7c02d766 6c031f7c95974521f17d6fedd429d35a1aea142b7cef5123f34504a79c21670b
lib/codeql/rust/elements/internal/generated/SynthConstructors.qll 8f7f08d1599e2dcbe553a7e4428a8787cef76b1d12b88f1b5925a4c59d617fae 8f7f08d1599e2dcbe553a7e4428a8787cef76b1d12b88f1b5925a4c59d617fae
lib/codeql/rust/elements/internal/generated/Synth.qll 839aa6b03b461ed9c79192d905969c63c99e7a12074e3487e74d2163d20ee499 44b8caba3186aaeb6d6b5759610627f8bc791e62001acfd1a66324086c17536f
lib/codeql/rust/elements/internal/generated/SynthConstructors.qll 3ceb5f6ee40b94955ce5f47feb454cc9129941aad3cdbe6e337bbe41e76a8a23 3ceb5f6ee40b94955ce5f47feb454cc9129941aad3cdbe6e337bbe41e76a8a23
lib/codeql/rust/elements/internal/generated/Token.qll 77a91a25ca5669703cf3a4353b591cef4d72caa6b0b9db07bb9e005d69c848d1 2fdffc4882ed3a6ca9ac6d1fb5f1ac5a471ca703e2ffdc642885fa558d6e373b
lib/codeql/rust/elements/internal/generated/TokenTree.qll 8577c2b097c1be2f0f7daa5acfcf146f78674a424d99563e08a84dd3e6d91b46 d2f30764e84dbfc0a6a5d3d8a5f935cd432413688cb32da9c94e420fbc10665c
lib/codeql/rust/elements/internal/generated/Trait.qll 8fa41b50fa0f68333534f2b66bb4ec8e103ff09ac8fa5c2cc64bc04beafec205 ce1c9aa6d0e2f05d28aab8e1165c3b9fb8e24681ade0cf6a9df2e8617abeae7e
@@ -592,7 +595,7 @@ lib/codeql/rust/elements/internal/generated/WhileExpr.qll 7edf1f23fbf953a2baabcd
lib/codeql/rust/elements/internal/generated/WildcardPat.qll d74b70b57a0a66bfae017a329352a5b27a6b9e73dd5521d627f680e810c6c59e 4b913b548ba27ff3c82fcd32cf996ff329cb57d176d3bebd0fcef394486ea499
lib/codeql/rust/elements/internal/generated/YeetExpr.qll cac328200872a35337b4bcb15c851afb4743f82c080f9738d295571eb01d7392 94af734eea08129b587fed849b643e7572800e8330c0b57d727d41abda47930b
lib/codeql/rust/elements/internal/generated/YieldExpr.qll 37e5f0c1e373a22bbc53d8b7f2c0e1f476e5be5080b8437c5e964f4e83fad79a 4a9a68643401637bf48e5c2b2f74a6bf0ddcb4ff76f6bffb61d436b685621e85
lib/codeql/rust/elements.qll 35959f2de54b6ee534fc592e1928c0829aa6e881e281a5acf724c10e4c685070 35959f2de54b6ee534fc592e1928c0829aa6e881e281a5acf724c10e4c685070
lib/codeql/rust/elements.qll a055d1f5bf70c9f8f6d5e34087146e813f3452b225b07b64f087d6e61f52a3b3 a055d1f5bf70c9f8f6d5e34087146e813f3452b225b07b64f087d6e61f52a3b3
test/extractor-tests/generated/Abi/Abi.ql 7f6e7dc4af86eca3ebdc79b10373988cd0871bd78b51997d3cffd969105e5fdd 2f936b6ca005c6157c755121584410c03e4a3949c23bee302fbe05ee10ce118f
test/extractor-tests/generated/Abi/Abi_getAbiString.ql a496762fcec5a0887b87023bbf93e9b650f02e20113e25c44d6e4281ae8f5335 14109c7ce11ba25e3cd6e7f1b3fcb4cb00622f2a4eac91bfe43145c5f366bc52
test/extractor-tests/generated/ArgList/ArgList.ql e412927756e72165d0e7c5c9bd3fca89d08197bbf760db8fb7683c64bb2229bc 043dba8506946fbb87753e22c387987d7eded6ddb963aa067f9e60ef9024d684

3
rust/ql/.gitattributes generated vendored
View File

@@ -228,6 +228,8 @@
/lib/codeql/rust/elements/internal/ExternItemImpl.qll linguist-generated
/lib/codeql/rust/elements/internal/ExternItemListConstructor.qll linguist-generated
/lib/codeql/rust/elements/internal/ExternItemListImpl.qll linguist-generated
/lib/codeql/rust/elements/internal/ExtractorStep.qll linguist-generated
/lib/codeql/rust/elements/internal/ExtractorStepConstructor.qll linguist-generated
/lib/codeql/rust/elements/internal/FieldExprConstructor.qll linguist-generated
/lib/codeql/rust/elements/internal/FieldListImpl.qll linguist-generated
/lib/codeql/rust/elements/internal/FnPtrTypeReprConstructor.qll linguist-generated
@@ -457,6 +459,7 @@
/lib/codeql/rust/elements/internal/generated/ExternCrate.qll linguist-generated
/lib/codeql/rust/elements/internal/generated/ExternItem.qll linguist-generated
/lib/codeql/rust/elements/internal/generated/ExternItemList.qll linguist-generated
/lib/codeql/rust/elements/internal/generated/ExtractorStep.qll linguist-generated
/lib/codeql/rust/elements/internal/generated/FieldExpr.qll linguist-generated
/lib/codeql/rust/elements/internal/generated/FieldList.qll linguist-generated
/lib/codeql/rust/elements/internal/generated/FnPtrTypeRepr.qll linguist-generated

19
rust/ql/integration-tests/conftest.py
View File

@@ -9,13 +9,26 @@ def cargo(cwd):
assert (cwd / "Cargo.toml").exists()
(cwd / "rust-project.json").unlink(missing_ok=True)
@pytest.fixture(scope="session")
def rust_sysroot_src() -> str:
rust_sysroot = pathlib.Path(commands.run("rustc --print sysroot", _capture=True))
ret = rust_sysroot.joinpath("lib", "rustlib", "src", "rust", "library")
assert ret.exists()
return str(ret)
@pytest.fixture
def rust_project(cwd):
def rust_project(cwd, rust_sysroot_src):
project_file = cwd / "rust-project.json"
assert project_file.exists()
rust_sysroot = pathlib.Path(commands.run("rustc --print sysroot", _capture=True))
project = json.loads(project_file.read_text())
project["sysroot_src"] = str(rust_sysroot.joinpath("lib", "rustlib", "src", "rust", "library"))
project["sysroot_src"] = rust_sysroot_src
project_file.write_text(json.dumps(project, indent=4))
(cwd / "Cargo.toml").unlink(missing_ok=True)
@pytest.fixture
def rust_check_diagnostics(check_diagnostics):
check_diagnostics.redact += [
"attributes.durations.*.ms",
"attributes.durations.*.pretty",
]
return check_diagnostics

39
rust/ql/integration-tests/hello-project/diagnostics.expected Normal file
View File

@@ -0,0 +1,39 @@
{
"attributes": {
"durations": {
"extract": {
"ms": "__REDACTED__",
"pretty": "__REDACTED__"
},
"loadManifest": {
"ms": "__REDACTED__",
"pretty": "__REDACTED__"
},
"loadSource": {
"ms": "__REDACTED__",
"pretty": "__REDACTED__"
},
"parse": {
"ms": "__REDACTED__",
"pretty": "__REDACTED__"
},
"total": {
"ms": "__REDACTED__",
"pretty": "__REDACTED__"
}
},
"numberOfFiles": 5,
"numberOfManifests": 1
},
"severity": "note",
"source": {
"extractorName": "rust",
"id": "rust/extractor/telemetry",
"name": "telemetry"
},
"visibility": {
"cliSummaryTable": false,
"statusPage": false,
"telemetry": true
}
}

15
rust/ql/integration-tests/hello-project/steps.cargo.expected Normal file
View File

@@ -0,0 +1,15 @@
| Cargo.toml:0:0:0:0 | LoadManifest(Cargo.toml) |
| src/directory_module/mod.rs:0:0:0:0 | Extract(src/directory_module/mod.rs) |
| src/directory_module/mod.rs:0:0:0:0 | LoadSource(src/directory_module/mod.rs) |
| src/directory_module/mod.rs:0:0:0:0 | Parse(src/directory_module/mod.rs) |
| src/directory_module/nested_module.rs:0:0:0:0 | Extract(src/directory_module/nested_module.rs) |
| src/directory_module/nested_module.rs:0:0:0:0 | LoadSource(src/directory_module/nested_module.rs) |
| src/directory_module/nested_module.rs:0:0:0:0 | Parse(src/directory_module/nested_module.rs) |
| src/directory_module/not_loaded.rs:0:0:0:0 | Extract(src/directory_module/not_loaded.rs) |
| src/directory_module/not_loaded.rs:0:0:0:0 | Parse(src/directory_module/not_loaded.rs) |
| src/file_module.rs:0:0:0:0 | Extract(src/file_module.rs) |
| src/file_module.rs:0:0:0:0 | LoadSource(src/file_module.rs) |
| src/file_module.rs:0:0:0:0 | Parse(src/file_module.rs) |
| src/main.rs:0:0:0:0 | Extract(src/main.rs) |
| src/main.rs:0:0:0:0 | LoadSource(src/main.rs) |
| src/main.rs:0:0:0:0 | Parse(src/main.rs) |

4
rust/ql/integration-tests/hello-project/steps.ql Normal file
View File

@@ -0,0 +1,4 @@
import codeql.rust.elements.internal.ExtractorStep
from ExtractorStep step
select step

15
rust/ql/integration-tests/hello-project/steps.rust-project.expected Normal file
View File

@@ -0,0 +1,15 @@
| rust-project.json:0:0:0:0 | LoadManifest(rust-project.json) |
| src/directory_module/mod.rs:0:0:0:0 | Extract(src/directory_module/mod.rs) |
| src/directory_module/mod.rs:0:0:0:0 | LoadSource(src/directory_module/mod.rs) |
| src/directory_module/mod.rs:0:0:0:0 | Parse(src/directory_module/mod.rs) |
| src/directory_module/nested_module.rs:0:0:0:0 | Extract(src/directory_module/nested_module.rs) |
| src/directory_module/nested_module.rs:0:0:0:0 | LoadSource(src/directory_module/nested_module.rs) |
| src/directory_module/nested_module.rs:0:0:0:0 | Parse(src/directory_module/nested_module.rs) |
| src/directory_module/not_loaded.rs:0:0:0:0 | Extract(src/directory_module/not_loaded.rs) |
| src/directory_module/not_loaded.rs:0:0:0:0 | Parse(src/directory_module/not_loaded.rs) |
| src/file_module.rs:0:0:0:0 | Extract(src/file_module.rs) |
| src/file_module.rs:0:0:0:0 | LoadSource(src/file_module.rs) |
| src/file_module.rs:0:0:0:0 | Parse(src/file_module.rs) |
| src/main.rs:0:0:0:0 | Extract(src/main.rs) |
| src/main.rs:0:0:0:0 | LoadSource(src/main.rs) |
| src/main.rs:0:0:0:0 | Parse(src/main.rs) |

2
rust/ql/integration-tests/hello-project/summary.expected
View File

@@ -1,4 +1,4 @@
| Elements extracted | 50 |
| Elements extracted | 65 |
| Elements unextracted | 0 |
| Extraction errors | 0 |
| Extraction warnings | 1 |

9
rust/ql/integration-tests/hello-project/test_project.py
View File

@@ -1,5 +1,10 @@
def test_cargo(codeql, rust, cargo, check_source_archive):
import pytest
@pytest.mark.ql_test("steps.ql", expected=".cargo.expected")
def test_cargo(codeql, rust, cargo, check_source_archive, rust_check_diagnostics):
codeql.database.create()
def test_rust_project(codeql, rust, rust_project, check_source_archive):
@pytest.mark.ql_test("steps.ql", expected=".rust-project.expected")
def test_rust_project(codeql, rust, rust_project, check_source_archive, rust_check_diagnostics):
codeql.database.create()

39
rust/ql/integration-tests/hello-workspace/diagnostics.cargo.expected Normal file
View File

@@ -0,0 +1,39 @@
{
"attributes": {
"durations": {
"extract": {
"ms": "__REDACTED__",
"pretty": "__REDACTED__"
},
"loadManifest": {
"ms": "__REDACTED__",
"pretty": "__REDACTED__"
},
"loadSource": {
"ms": "__REDACTED__",
"pretty": "__REDACTED__"
},
"parse": {
"ms": "__REDACTED__",
"pretty": "__REDACTED__"
},
"total": {
"ms": "__REDACTED__",
"pretty": "__REDACTED__"
}
},
"numberOfFiles": 4,
"numberOfManifests": 2
},
"severity": "note",
"source": {
"extractorName": "rust",
"id": "rust/extractor/telemetry",
"name": "telemetry"
},
"visibility": {
"cliSummaryTable": false,
"statusPage": false,
"telemetry": true
}
}

39
rust/ql/integration-tests/hello-workspace/diagnostics.rust-project.expected Normal file
View File

@@ -0,0 +1,39 @@
{
"attributes": {
"durations": {
"extract": {
"ms": "__REDACTED__",
"pretty": "__REDACTED__"
},
"loadManifest": {
"ms": "__REDACTED__",
"pretty": "__REDACTED__"
},
"loadSource": {
"ms": "__REDACTED__",
"pretty": "__REDACTED__"
},
"parse": {
"ms": "__REDACTED__",
"pretty": "__REDACTED__"
},
"total": {
"ms": "__REDACTED__",
"pretty": "__REDACTED__"
}
},
"numberOfFiles": 4,
"numberOfManifests": 1
},
"severity": "note",
"source": {
"extractorName": "rust",
"id": "rust/extractor/telemetry",
"name": "telemetry"
},
"visibility": {
"cliSummaryTable": false,
"statusPage": false,
"telemetry": true
}
}

14
rust/ql/integration-tests/hello-workspace/steps.cargo.expected Normal file
View File

@@ -0,0 +1,14 @@
| exe/Cargo.toml:0:0:0:0 | LoadManifest(exe/Cargo.toml) |
| exe/src/a_module.rs:0:0:0:0 | Extract(exe/src/a_module.rs) |
| exe/src/a_module.rs:0:0:0:0 | LoadSource(exe/src/a_module.rs) |
| exe/src/a_module.rs:0:0:0:0 | Parse(exe/src/a_module.rs) |
| exe/src/main.rs:0:0:0:0 | Extract(exe/src/main.rs) |
| exe/src/main.rs:0:0:0:0 | LoadSource(exe/src/main.rs) |
| exe/src/main.rs:0:0:0:0 | Parse(exe/src/main.rs) |
| lib/Cargo.toml:0:0:0:0 | LoadManifest(lib/Cargo.toml) |
| lib/src/a_module/mod.rs:0:0:0:0 | Extract(lib/src/a_module/mod.rs) |
| lib/src/a_module/mod.rs:0:0:0:0 | LoadSource(lib/src/a_module/mod.rs) |
| lib/src/a_module/mod.rs:0:0:0:0 | Parse(lib/src/a_module/mod.rs) |
| lib/src/lib.rs:0:0:0:0 | Extract(lib/src/lib.rs) |
| lib/src/lib.rs:0:0:0:0 | LoadSource(lib/src/lib.rs) |
| lib/src/lib.rs:0:0:0:0 | Parse(lib/src/lib.rs) |

4
rust/ql/integration-tests/hello-workspace/steps.ql Normal file
View File

@@ -0,0 +1,4 @@
import codeql.rust.elements.internal.ExtractorStep
from ExtractorStep step
select step

13
rust/ql/integration-tests/hello-workspace/steps.rust-project.expected Normal file
View File

@@ -0,0 +1,13 @@
| exe/src/a_module.rs:0:0:0:0 | Extract(exe/src/a_module.rs) |
| exe/src/a_module.rs:0:0:0:0 | LoadSource(exe/src/a_module.rs) |
| exe/src/a_module.rs:0:0:0:0 | Parse(exe/src/a_module.rs) |
| exe/src/main.rs:0:0:0:0 | Extract(exe/src/main.rs) |
| exe/src/main.rs:0:0:0:0 | LoadSource(exe/src/main.rs) |
| exe/src/main.rs:0:0:0:0 | Parse(exe/src/main.rs) |
| lib/src/a_module/mod.rs:0:0:0:0 | Extract(lib/src/a_module/mod.rs) |
| lib/src/a_module/mod.rs:0:0:0:0 | LoadSource(lib/src/a_module/mod.rs) |
| lib/src/a_module/mod.rs:0:0:0:0 | Parse(lib/src/a_module/mod.rs) |
| lib/src/lib.rs:0:0:0:0 | Extract(lib/src/lib.rs) |
| lib/src/lib.rs:0:0:0:0 | LoadSource(lib/src/lib.rs) |
| lib/src/lib.rs:0:0:0:0 | Parse(lib/src/lib.rs) |
| rust-project.json:0:0:0:0 | LoadManifest(rust-project.json) |

2
rust/ql/integration-tests/hello-workspace/summary.expected → rust/ql/integration-tests/hello-workspace/summary.cargo.expected
View File

@@ -1,4 +1,4 @@
| Elements extracted | 72 |
| Elements extracted | 86 |
| Elements unextracted | 0 |
| Extraction errors | 0 |
| Extraction warnings | 0 |

17
rust/ql/integration-tests/hello-workspace/summary.rust-project.expected Normal file
View File

@@ -0,0 +1,17 @@
| Elements extracted | 85 |
| Elements unextracted | 0 |
| Extraction errors | 0 |
| Extraction warnings | 0 |
| Files extracted - total | 4 |
| Files extracted - with errors | 0 |
| Files extracted - without errors | 4 |
| Inconsistencies - AST | 0 |
| Inconsistencies - CFG | 0 |
| Inconsistencies - data flow | 0 |
| Lines of code extracted | 9 |
| Lines of user code extracted | 9 |
| Macro calls - resolved | 2 |
| Macro calls - total | 2 |
| Macro calls - unresolved | 0 |
| Taint sources - active | 0 |
| Taint sources - total | 0 |

13
rust/ql/integration-tests/hello-workspace/test_workspace.py
View File

@@ -1,5 +1,14 @@
def test_cargo(codeql, rust, cargo, check_source_archive):
import pytest
@pytest.mark.ql_test("steps.ql", expected=".cargo.expected")
@pytest.mark.ql_test("summary.qlref", expected=".cargo.expected")
def test_cargo(codeql, rust, cargo, check_source_archive, rust_check_diagnostics):
rust_check_diagnostics.expected_suffix = ".cargo.expected"
codeql.database.create()
def test_rust_project(codeql, rust, rust_project, check_source_archive):
@pytest.mark.ql_test("steps.ql", expected=".rust-project.expected")
@pytest.mark.ql_test("summary.qlref", expected=".rust-project.expected")
def test_rust_project(codeql, rust, rust_project, check_source_archive, rust_check_diagnostics):
rust_check_diagnostics.expected_suffix = ".rust-project.expected"
codeql.database.create()

16
rust/ql/lib/codeql/files/FileSystem.qll
View File

@@ -6,6 +6,7 @@ private import codeql.rust.elements.SourceFile
private import codeql.rust.elements.AstNode
private import codeql.rust.elements.Comment
private import codeql.rust.Diagnostics
private import codeql.rust.elements.internal.ExtractorStep
private module Input implements InputSig {
abstract class ContainerBase extends @container {
@@ -36,7 +37,9 @@ class Folder = Impl::Folder;
/** A file. */
class File extends Container, Impl::File {
/** Holds if this file was extracted from ordinary source code. */
predicate fromSource() { any() }
predicate fromSource() {
exists(ExtractorStep s | s.getAction() = "Extract" and s.getFile() = this)
}
/**
* Gets the number of lines containing code in this file. This value
@@ -58,11 +61,20 @@ class File extends Container, Impl::File {
}
}
/**
* A source file that was extracted.
*
* TODO: rename `SourceFile` from the generated AST to give that name to this class.
*/
class ExtractedFile extends File {
ExtractedFile() { this.fromSource() }
}
/**
* A successfully extracted file, that is, a file that was extracted and
* contains no extraction errors or warnings.
*/
class SuccessfullyExtractedFile extends File {
class SuccessfullyExtractedFile extends ExtractedFile {
SuccessfullyExtractedFile() {
not exists(Diagnostic d |
d.getLocation().getFile() = this and

1
rust/ql/lib/codeql/rust/elements.qll generated
View File

@@ -3,6 +3,7 @@
* This module exports all modules providing `Element` subclasses.
*/
import codeql.files.FileSystem
import codeql.rust.elements.Abi
import codeql.rust.elements.Addressable
import codeql.rust.elements.ArgList

13
rust/ql/lib/codeql/rust/elements/internal/ExtractorStep.qll generated Normal file
View File

@@ -0,0 +1,13 @@
// generated by codegen, do not edit
/**
* This module provides the class `ExtractorStep`.
*/
private import ExtractorStepImpl
import codeql.rust.elements.Element
import codeql.files.FileSystem
/**
* INTERNAL: Do not use.
*/
final class ExtractorStep = Impl::ExtractorStep;

14
rust/ql/lib/codeql/rust/elements/internal/ExtractorStepConstructor.qll generated Normal file
View File

@@ -0,0 +1,14 @@
// generated by codegen, remove this comment if you wish to edit this file
/**
* This module defines the hook used internally to tweak the characteristic predicate of
* `ExtractorStep` synthesized instances.
* INTERNAL: Do not use.
*/
private import codeql.rust.elements.internal.generated.Raw
/**
* The characteristic predicate of `ExtractorStep` synthesized instances.
* INTERNAL: Do not use.
*/
predicate constructExtractorStep(Raw::ExtractorStep id) { any() }

32
rust/ql/lib/codeql/rust/elements/internal/ExtractorStepImpl.qll Normal file
View File

@@ -0,0 +1,32 @@
/**
* This module provides a hand-modifiable wrapper around the generated class `ExtractorStep`.
*
* INTERNAL: Do not use.
*/
private import codeql.rust.elements.internal.generated.ExtractorStep
/**
* INTERNAL: This module contains the customizable definition of `ExtractorStep` and should not
* be referenced directly.
*/
module Impl {
class ExtractorStep extends Generated::ExtractorStep {
override string toString() {
result = this.getAction() + "(" + this.getFile().getAbsolutePath() + ")"
}
/**
* Provides location information for this step.
*/
predicate hasLocationInfo(
string filepath, int startline, int startcolumn, int endline, int endcolumn
) {
this.getFile().getAbsolutePath() = filepath and
startline = 0 and
startcolumn = 0 and
endline = 0 and
endcolumn = 0
}
}
}

45
rust/ql/lib/codeql/rust/elements/internal/generated/ExtractorStep.qll generated Normal file
View File

@@ -0,0 +1,45 @@
// generated by codegen, do not edit
/**
* This module provides the generated definition of `ExtractorStep`.
* INTERNAL: Do not import directly.
*/
private import codeql.rust.elements.internal.generated.Synth
private import codeql.rust.elements.internal.generated.Raw
import codeql.rust.elements.internal.ElementImpl::Impl as ElementImpl
import codeql.files.FileSystem
/**
* INTERNAL: This module contains the fully generated definition of `ExtractorStep` and should not
* be referenced directly.
*/
module Generated {
/**
* INTERNAL: Do not reference the `Generated::ExtractorStep` class directly.
* Use the subclass `ExtractorStep`, where the following predicates are available.
*/
class ExtractorStep extends Synth::TExtractorStep, ElementImpl::Element {
override string getAPrimaryQlClass() { result = "ExtractorStep" }
/**
* Gets the action of this extractor step.
*/
string getAction() {
result = Synth::convertExtractorStepToRaw(this).(Raw::ExtractorStep).getAction()
}
/**
* Gets the file of this extractor step.
*/
File getFile() {
result = Synth::convertExtractorStepToRaw(this).(Raw::ExtractorStep).getFile()
}
/**
* Gets the duration ms of this extractor step.
*/
int getDurationMs() {
result = Synth::convertExtractorStepToRaw(this).(Raw::ExtractorStep).getDurationMs()
}
}
}

18
rust/ql/lib/codeql/rust/elements/internal/generated/ParentChild.qll generated
View File

@@ -5,12 +5,28 @@
import codeql.rust.elements
import codeql.rust.elements.internal.ArrayExprInternal
import codeql.rust.elements.internal.ExtractorStep
private module Impl {
private Element getImmediateChildOfElement(Element e, int index, string partialPredicateCall) {
none()
}
private Element getImmediateChildOfExtractorStep(
ExtractorStep e, int index, string partialPredicateCall
) {
exists(int b, int bElement, int n |
b = 0 and
bElement = b + 1 + max(int i | i = -1 or exists(getImmediateChildOfElement(e, i, _)) | i) and
n = bElement and
(
none()
or
result = getImmediateChildOfElement(e, index - b, partialPredicateCall)
)
)
}
private Element getImmediateChildOfLocatable(Locatable e, int index, string partialPredicateCall) {
exists(int b, int bElement, int n |
b = 0 and
@@ -3705,6 +3721,8 @@ private module Impl {
// * none() simplifies generation, as we can append `or ...` without a special case for the first item
none()
or
result = getImmediateChildOfExtractorStep(e, index, partialAccessor)
or
result = getImmediateChildOfFormat(e, index, partialAccessor)
or
result = getImmediateChildOfFormatArgument(e, index, partialAccessor)

24
rust/ql/lib/codeql/rust/elements/internal/generated/Raw.qll generated
View File

@@ -3,6 +3,8 @@
* This module holds thin fully generated class definitions around DB entities.
*/
module Raw {
private import codeql.files.FileSystem
/**
* INTERNAL: Do not use.
*/
@@ -10,6 +12,28 @@ module Raw {
string toString() { none() }
}
/**
* INTERNAL: Do not use.
*/
class ExtractorStep extends @extractor_step, Element {
override string toString() { result = "ExtractorStep" }
/**
* Gets the action of this extractor step.
*/
string getAction() { extractor_steps(this, result, _, _) }
/**
* Gets the file of this extractor step.
*/
File getFile() { extractor_steps(this, _, result, _) }
/**
* Gets the duration ms of this extractor step.
*/
int getDurationMs() { extractor_steps(this, _, _, result) }
}
/**
* INTERNAL: Do not use.
*/

20
rust/ql/lib/codeql/rust/elements/internal/generated/Synth.qll generated
View File

@@ -142,6 +142,10 @@ module Synth {
* INTERNAL: Do not use.
*/
TExternItemList(Raw::ExternItemList id) { constructExternItemList(id) } or
/**
* INTERNAL: Do not use.
*/
TExtractorStep(Raw::ExtractorStep id) { constructExtractorStep(id) } or
/**
* INTERNAL: Do not use.
*/
@@ -953,6 +957,12 @@ module Synth {
*/
TExternItemList convertExternItemListFromRaw(Raw::Element e) { result = TExternItemList(e) }
/**
* INTERNAL: Do not use.
* Converts a raw element to a synthesized `TExtractorStep`, if possible.
*/
TExtractorStep convertExtractorStepFromRaw(Raw::Element e) { result = TExtractorStep(e) }
/**
* INTERNAL: Do not use.
* Converts a raw element to a synthesized `TFieldExpr`, if possible.
@@ -1842,6 +1852,8 @@ module Synth {
* Converts a raw DB element to a synthesized `TElement`, if possible.
*/
TElement convertElementFromRaw(Raw::Element e) {
result = convertExtractorStepFromRaw(e)
or
result = convertLocatableFromRaw(e)
or
result = convertUnextractedFromRaw(e)
@@ -2367,6 +2379,12 @@ module Synth {
*/
Raw::Element convertExternItemListToRaw(TExternItemList e) { e = TExternItemList(result) }
/**
* INTERNAL: Do not use.
* Converts a synthesized `TExtractorStep` to a raw DB element, if possible.
*/
Raw::Element convertExtractorStepToRaw(TExtractorStep e) { e = TExtractorStep(result) }
/**
* INTERNAL: Do not use.
* Converts a synthesized `TFieldExpr` to a raw DB element, if possible.
@@ -3254,6 +3272,8 @@ module Synth {
* Converts a synthesized `TElement` to a raw DB element, if possible.
*/
Raw::Element convertElementToRaw(TElement e) {
result = convertExtractorStepToRaw(e)
or
result = convertLocatableToRaw(e)
or
result = convertUnextractedToRaw(e)

1
rust/ql/lib/codeql/rust/elements/internal/generated/SynthConstructors.qll generated
View File

@@ -35,6 +35,7 @@ import codeql.rust.elements.internal.ExprStmtConstructor
import codeql.rust.elements.internal.ExternBlockConstructor
import codeql.rust.elements.internal.ExternCrateConstructor
import codeql.rust.elements.internal.ExternItemListConstructor
import codeql.rust.elements.internal.ExtractorStepConstructor
import codeql.rust.elements.internal.FieldExprConstructor
import codeql.rust.elements.internal.FnPtrTypeReprConstructor
import codeql.rust.elements.internal.ForExprConstructor

10
rust/ql/lib/rust.dbscheme
View File

@@ -120,10 +120,18 @@ locatable_locations(
// from schema
@element =
@locatable
@extractor_step
| @locatable
| @unextracted
;
extractor_steps(
unique int id: @extractor_step,
string action: string ref,
int file: @file ref,
int duration_ms: int ref
);
@locatable =
@ast_node
;

2
rust/ql/src/queries/diagnostics/ExtractedFiles.ql
View File

@@ -8,6 +8,6 @@
import rust
from File f
from ExtractedFile f
where exists(f.getRelativePath())
select f, "File successfully extracted."

2
rust/ql/src/queries/diagnostics/ExtractionErrors.ql
View File

@@ -11,7 +11,7 @@ import codeql.files.FileSystem
/** Gets the SARIF severity to associate with an error. */
int getSeverity() { result = 2 }
from ExtractionError error, File f
from ExtractionError error, ExtractedFile f
where
f = error.getLocation().getFile() and
exists(f.getRelativePath())

7
rust/ql/src/queries/summary/SummaryStats.ql
View File

@@ -21,10 +21,13 @@ where
or
key = "Extraction warnings" and value = count(ExtractionWarning w)
or
key = "Files extracted - total" and value = count(File f | exists(f.getRelativePath()))
key = "Files extracted - total" and value = count(ExtractedFile f | exists(f.getRelativePath()))
or
key = "Files extracted - with errors" and
value = count(File f | exists(f.getRelativePath()) and not f instanceof SuccessfullyExtractedFile)
value =
count(ExtractedFile f |
exists(f.getRelativePath()) and not f instanceof SuccessfullyExtractedFile
)
or
key = "Files extracted - without errors" and
value = count(SuccessfullyExtractedFile f | exists(f.getRelativePath()))

7
rust/ql/test/extractor-tests/File/File.expected
View File

@@ -1,3 +1,4 @@
| a_file.rs:0:0:0:0 | a_file.rs |
| another_file.rs:0:0:0:0 | another_file.rs |
| lib.rs:0:0:0:0 | lib.rs |
| Cargo.toml:0:0:0:0 | Cargo.toml | fromSource: no |
| a_file.rs:0:0:0:0 | a_file.rs | fromSource: yes |
| another_file.rs:0:0:0:0 | another_file.rs | fromSource: yes |
| lib.rs:0:0:0:0 | lib.rs | fromSource: yes |

8
rust/ql/test/extractor-tests/File/File.ql
View File

@@ -1,5 +1,7 @@
import rust
from File f
where exists(f.getRelativePath())
select f
from File f, string fromSource
where
exists(f.getRelativePath()) and
if f.fromSource() then fromSource = "fromSource: yes" else fromSource = "fromSource: no"
select f, fromSource

1
rust/ql/test/query-tests/diagnostics/ExtractedFiles.expected
View File

@@ -5,4 +5,3 @@
| main.rs:0:0:0:0 | main.rs | File successfully extracted. |
| my_macro.rs:0:0:0:0 | my_macro.rs | File successfully extracted. |
| my_struct.rs:0:0:0:0 | my_struct.rs | File successfully extracted. |
| options.yml:0:0:0:0 | options.yml | File successfully extracted. |

1
rust/ql/test/query-tests/diagnostics/LinesOfUserCodeInFiles.expected
View File

@@ -5,4 +5,5 @@
| lib.rs:0:0:0:0 | lib.rs | 5 |
| does_not_compile.rs:0:0:0:0 | does_not_compile.rs | 3 |
| error.rs:0:0:0:0 | error.rs | 3 |
| Cargo.toml:0:0:0:0 | Cargo.toml | 0 |
| options.yml:0:0:0:0 | options.yml | 0 |

6
rust/ql/test/query-tests/diagnostics/SummaryStats.expected
View File

@@ -1,10 +1,10 @@
| Elements extracted | 382 |
| Elements extracted | 404 |
| Elements unextracted | 0 |
| Extraction errors | 0 |
| Extraction warnings | 7 |
| Files extracted - total | 8 |
| Files extracted - total | 7 |
| Files extracted - with errors | 3 |
| Files extracted - without errors | 5 |
| Files extracted - without errors | 4 |
| Inconsistencies - AST | 0 |
| Inconsistencies - CFG | 0 |
| Inconsistencies - data flow | 0 |

9
rust/schema/prelude.py
View File

@@ -3,6 +3,7 @@ from misc.codegen.lib.schemadefs import *
include("../shared/tree-sitter-extractor/src/generator/prefix.dbscheme")
include("prefix.dbscheme")
File = imported("File", "codeql.files.FileSystem")
@qltest.skip
class Element:
@@ -93,3 +94,11 @@ class Resolvable(AstNode):
"""
resolved_path: optional[string] | rust.detach | ql.internal
resolved_crate_origin: optional[string] | rust.detach | ql.internal
@qltest.skip
@ql.internal
class ExtractorStep(Element):
action: string
file: File
duration_ms: int