Rust: aggregate projects into workspaces

rust/extractor/Cargo.toml

@@ -36,3 +36,4 @@ glob = "0.3.2"
 chrono = { version = "0.4.39", features = ["serde"] }
 serde_json = "1.0.135"
 dunce = "1.0.5"
+toml = "0.8.19"

rust/extractor/src/diagnostics.rs

@@ -79,6 +79,7 @@ pub struct Diagnostics<T> {
 pub enum ExtractionStepKind {
     #[default]
     LoadManifest,
+    FindManifests,
     LoadSource,
     Parse,
     Extract,
@@ -88,12 +89,12 @@ pub enum ExtractionStepKind {
 #[serde(rename_all = "camelCase")]
 pub struct ExtractionStep {
     pub action: ExtractionStepKind,
-    pub file: PathBuf,
+    pub file: Option<PathBuf>,
     pub ms: u128,
 }
 
 impl ExtractionStep {
-    fn new(start: Instant, action: ExtractionStepKind, file: PathBuf) -> Self {
+    fn new(start: Instant, action: ExtractionStepKind, file: Option<PathBuf>) -> Self {
         let ret = ExtractionStep {
             action,
             file,
@@ -107,20 +108,36 @@ impl ExtractionStep {
         Self::new(
             start,
             ExtractionStepKind::LoadManifest,
-            PathBuf::from(target.manifest_path()),
+            Some(PathBuf::from(target.manifest_path())),
         )
     }
 
     pub fn parse(start: Instant, target: &Path) -> Self {
-        Self::new(start, ExtractionStepKind::Parse, PathBuf::from(target))
+        Self::new(
+            start,
+            ExtractionStepKind::Parse,
+            Some(PathBuf::from(target)),
+        )
     }
 
     pub fn extract(start: Instant, target: &Path) -> Self {
-        Self::new(start, ExtractionStepKind::Extract, PathBuf::from(target))
+        Self::new(
+            start,
+            ExtractionStepKind::Extract,
+            Some(PathBuf::from(target)),
+        )
     }
 
     pub fn load_source(start: Instant, target: &Path) -> Self {
-        Self::new(start, ExtractionStepKind::LoadSource, PathBuf::from(target))
+        Self::new(
+            start,
+            ExtractionStepKind::LoadSource,
+            Some(PathBuf::from(target)),
+        )
+    }
+
+    pub fn find_manifests(start: Instant) -> Self {
+        Self::new(start, ExtractionStepKind::FindManifests, None)
     }
 }
 

rust/extractor/src/generated/.generated.list

@@ -1,2 +1,2 @@
 mod.rs 4bcb9def847469aae9d8649461546b7c21ec97cf6e63d3cf394e339915ce65d7 4bcb9def847469aae9d8649461546b7c21ec97cf6e63d3cf394e339915ce65d7
-top.rs f042175b9f2dc4092ed04dde2073735560a9c3050a2f07ee79193c492e91cabc f042175b9f2dc4092ed04dde2073735560a9c3050a2f07ee79193c492e91cabc
+top.rs 97b9c3c5485196cc7949ec1b67c5844b7ff7af67bc2339276adbdafb03789ac0 97b9c3c5485196cc7949ec1b67c5844b7ff7af67bc2339276adbdafb03789ac0

rust/extractor/src/generated/top.rs

@@ -26,7 +26,7 @@ impl trap::TrapClass for Element {
 pub struct ExtractorStep {
     pub id: trap::TrapId<ExtractorStep>,
     pub action: String,
-    pub file: trap::Label<File>,
+    pub file: Option<trap::Label<File>>,
     pub duration_ms: usize,
 }
 
@@ -36,7 +36,10 @@ impl trap::TrapEntry for ExtractorStep {
     }
 
     fn emit(self, id: trap::Label<Self>, out: &mut trap::Writer) {
-        out.add_tuple("extractor_steps", vec![id.into(), self.action.into(), self.file.into(), self.duration_ms.into()]);
+        out.add_tuple("extractor_steps", vec![id.into(), self.action.into(), self.duration_ms.into()]);
+        if let Some(v) = self.file {
+            out.add_tuple("extractor_step_files", vec![id.into(), v.into()]);
+        }
     }
 }
 

rust/extractor/src/main.rs

@@ -145,7 +145,7 @@ impl<'a> Extractor<'a> {
         emit_extraction_diagnostics(start, cfg, &self.steps)?;
         let mut trap = self.traps.create("diagnostics", "extraction");
         for step in self.steps {
-            let file = trap.emit_file(&step.file);
+            let file = step.file.as_ref().map(|f| trap.emit_file(f));
             let duration_ms = usize::try_from(step.ms).unwrap_or_else(|_e| {
                 warn!("extraction step duration overflowed ({step:?})");
                 i32::MAX as usize
@@ -160,6 +160,13 @@ impl<'a> Extractor<'a> {
         trap.commit()?;
         Ok(())
     }
+
+    pub fn find_manifests(&mut self, files: &[PathBuf]) -> anyhow::Result<Vec<ProjectManifest>> {
+        let before = Instant::now();
+        let ret = rust_analyzer::find_project_manifests(files);
+        self.steps.push(ExtractionStep::find_manifests(before));
+        ret
+    }
 }
 
 fn cwd() -> anyhow::Result<AbsPathBuf> {
@@ -199,7 +206,7 @@ fn main() -> anyhow::Result<()> {
             dunce::canonicalize(&file).unwrap_or(file)
         })
         .collect();
-    let manifests = rust_analyzer::find_project_manifests(&files)?;
+    let manifests = extractor.find_manifests(&files)?;
     let mut map: HashMap<&Path, (&ProjectManifest, Vec<&Path>)> = manifests
         .iter()
         .map(|x| (x.manifest_path().parent().as_ref(), (x, Vec::new())))

rust/extractor/src/rust_analyzer.rs

@@ -1,12 +1,12 @@
 use itertools::Itertools;
-use log::{debug, info};
+use log::{debug, error, info};
 use ra_ap_base_db::SourceDatabase;
 use ra_ap_hir::Semantics;
 use ra_ap_ide_db::RootDatabase;
 use ra_ap_load_cargo::{load_workspace_at, LoadCargoConfig, ProcMacroServerChoice};
 use ra_ap_paths::Utf8PathBuf;
-use ra_ap_project_model::CargoConfig;
 use ra_ap_project_model::ProjectManifest;
+use ra_ap_project_model::{CargoConfig, ManifestPath};
 use ra_ap_span::Edition;
 use ra_ap_span::EditionedFileId;
 use ra_ap_span::TextRange;
@@ -17,7 +17,10 @@ use ra_ap_vfs::Vfs;
 use ra_ap_vfs::VfsPath;
 use ra_ap_vfs::{AbsPathBuf, FileId};
 use std::borrow::Cow;
+use std::collections::{HashMap, HashSet};
+use std::fs;
 use std::path::{Path, PathBuf};
+use std::rc::Rc;
 use triomphe::Arc;
 
 pub enum RustAnalyzer<'a> {
@@ -128,6 +131,76 @@ impl<'a> RustAnalyzer<'a> {
     }
 }
 
+struct ToMlReader {
+    cache: HashMap<ManifestPath, Rc<toml::Table>>,
+}
+
+impl ToMlReader {
+    fn new() -> Self {
+        Self {
+            cache: HashMap::new(),
+        }
+    }
+
+    fn read(&mut self, manifest: &ManifestPath) -> anyhow::Result<Rc<toml::Table>> {
+        if let Some(table) = self.cache.get(manifest) {
+            return Ok(table.clone());
+        }
+        let content = fs::read_to_string(manifest).map_err(|e| {
+            error!("failed to read {} ({e})", manifest.as_str());
+            e
+        })?;
+        let table = Rc::<toml::Table>::new(content.parse().map_err(|e| {
+            error!("failed to parse {} ({e})", manifest.as_str());
+            e
+        })?);
+        self.cache.insert(manifest.clone(), table.clone());
+        Ok(table)
+    }
+}
+
+fn find_workspace(
+    reader: &mut ToMlReader,
+    manifest: &ProjectManifest,
+) -> anyhow::Result<ProjectManifest> {
+    let ProjectManifest::CargoToml(cargo) = manifest else {
+        return Err(anyhow::anyhow!("{manifest} not a cargo manifest"));
+    };
+    let toml = reader.read(cargo)?;
+    if toml.contains_key("workspace") {
+        return Ok(manifest.clone());
+    }
+    let Some(parent_dir) = cargo.parent().parent() else {
+        return Err(anyhow::anyhow!("no parent dir for {cargo}"));
+    };
+    let discovered = ProjectManifest::discover(parent_dir)?;
+    discovered
+        .iter()
+        .filter_map(|it| match it {
+            ProjectManifest::CargoToml(other)
+                if cargo.starts_with(other.parent())
+                    && reader.read(other).is_ok_and(|it| {
+                        it.get("workspace")
+                            .and_then(|w| w.as_table())
+                            .and_then(|t| t.get("members"))
+                            .and_then(|ms| ms.as_array())
+                            .is_some_and(|ms| {
+                                ms.iter().any(|m| {
+                                    m.as_str()
+                                        .is_some_and(|s| other.parent().join(s) == cargo.parent())
+                                })
+                            })
+                    }) =>
+            {
+                debug!("found workspace {other} containing {cargo}");
+                Some(it.clone())
+            }
+            _ => None,
+        })
+        .next()
+        .ok_or(anyhow::anyhow!("no workspace found for {manifest}"))
+}
+
 pub fn find_project_manifests(
     files: &[PathBuf],
 ) -> anyhow::Result<Vec<ra_ap_project_model::ProjectManifest>> {
@@ -136,7 +209,13 @@ pub fn find_project_manifests(
         .iter()
         .map(|path| AbsPathBuf::assert_utf8(current.join(path)))
         .collect();
-    let ret = ra_ap_project_model::ProjectManifest::discover_all(&abs_files);
+    let discovered = ra_ap_project_model::ProjectManifest::discover_all(&abs_files);
+    let mut ret = HashSet::new();
+    let mut reader = ToMlReader::new();
+    for manifest in discovered {
+        let workspace = find_workspace(&mut reader, &manifest).unwrap_or(manifest);
+        ret.insert(workspace);
+    }
     let iter = || ret.iter().map(|m| format!("  {m}"));
     const LOG_LIMIT: usize = 10;
     if ret.len() <= LOG_LIMIT {
@@ -152,8 +231,9 @@ pub fn find_project_manifests(
             iter().dropping(LOG_LIMIT).join("\n")
         );
     }
-    Ok(ret)
+    Ok(ret.into_iter().collect())
 }
+
 fn from_utf8_lossy(v: &[u8]) -> (Cow<'_, str>, Option<SyntaxError>) {
     let mut iter = v.utf8_chunks();
     let (first_valid, first_invalid) = if let Some(chunk) = iter.next() {