Rust: fix macro expansion in library code

There was a mismatch between a `self.macro_context_level += 1` and the
corresponding `self.macro_context_level -= 1`, which resulted in an
`usize` underflow (panic in debug mode, wrong behaviour in release
mode).

This fixes it and adds a relevant assertion and test. In order to
properly test library mode extraction, a special option enforcing that
on source code as well is added.

rust/extractor/src/config.rs

@@ -71,6 +71,7 @@ pub struct Config {
     pub proc_macro_server: Option<PathBuf>,
     pub skip_path_resolution: bool,
     pub extract_dependencies_as_source: bool,
+    pub force_library_mode: bool, // for testing purposes
 }
 
 impl Config {

rust/extractor/src/main.rs

@@ -293,6 +293,11 @@ fn main() -> anyhow::Result<()> {
     } else {
         (SourceKind::Library, ResolvePaths::No)
     };
+    let (source_mode, source_resolve_paths) = if cfg.force_library_mode {
+        (library_mode, library_resolve_paths)
+    } else {
+        (SourceKind::Source, resolve_paths)
+    };
     let mut processed_files: HashSet<PathBuf, RandomState> =
         HashSet::from_iter(files.iter().cloned());
     for (manifest, files) in map.values().filter(|(_, files)| !files.is_empty()) {
@@ -311,12 +316,10 @@ fn main() -> anyhow::Result<()> {
                         file,
                         &semantics,
                         vfs,
-                        resolve_paths,
-                        SourceKind::Source,
+                        source_resolve_paths,
+                        source_mode,
                     ),
-                    Err(reason) => {
-                        extractor.extract_without_semantics(file, SourceKind::Source, &reason)
-                    }
+                    Err(reason) => extractor.extract_without_semantics(file, source_mode, &reason),
                 };
             }
             for (file_id, file) in vfs.iter() {

rust/extractor/src/qltest.rs

@@ -19,7 +19,7 @@ fn dump_lib() -> anyhow::Result<()> {
         .iter()
         .map(|p| p.file_stem().expect("results of glob must have a name"))
         .filter(|&p| !["main", "lib", "proc_macro"].map(OsStr::new).contains(&p))
-        .map(|p| format!("mod {};", p.to_string_lossy()))
+        .map(|p| format!("pub mod {};", p.to_string_lossy()))
         .join("\n");
     fs::write("lib.rs", lib).context("writing lib.rs")
 }

rust/extractor/src/translate/base.rs

@@ -24,33 +24,31 @@ use ra_ap_syntax::{
 
 impl Emission<ast::Item> for Translator<'_> {
     fn pre_emit(&mut self, node: &ast::Item) -> Option<Label<generated::Item>> {
-        self.prepare_item_expansion(node).map(Into::into)
+        self.item_pre_emit(node).map(Into::into)
     }
 
     fn post_emit(&mut self, node: &ast::Item, label: Label<generated::Item>) {
-        self.emit_item_expansion(node, label);
+        self.item_post_emit(node, label);
     }
 }
 
 impl Emission<ast::AssocItem> for Translator<'_> {
     fn pre_emit(&mut self, node: &ast::AssocItem) -> Option<Label<generated::AssocItem>> {
-        self.prepare_item_expansion(&node.clone().into())
-            .map(Into::into)
+        self.item_pre_emit(&node.clone().into()).map(Into::into)
     }
 
     fn post_emit(&mut self, node: &ast::AssocItem, label: Label<generated::AssocItem>) {
-        self.emit_item_expansion(&node.clone().into(), label.into());
+        self.item_post_emit(&node.clone().into(), label.into());
     }
 }
 
 impl Emission<ast::ExternItem> for Translator<'_> {
     fn pre_emit(&mut self, node: &ast::ExternItem) -> Option<Label<generated::ExternItem>> {
-        self.prepare_item_expansion(&node.clone().into())
-            .map(Into::into)
+        self.item_pre_emit(&node.clone().into()).map(Into::into)
     }
 
     fn post_emit(&mut self, node: &ast::ExternItem, label: Label<generated::ExternItem>) {
-        self.emit_item_expansion(&node.clone().into(), label.into());
+        self.item_post_emit(&node.clone().into(), label.into());
     }
 }
 
@@ -849,35 +847,6 @@ impl<'a> Translator<'a> {
             })
     }
 
-    pub(crate) fn prepare_item_expansion(
-        &mut self,
-        node: &ast::Item,
-    ) -> Option<Label<generated::MacroCall>> {
-        if self.source_kind == SourceKind::Library {
-            // if the item expands via an attribute macro, we want to only emit the expansion
-            if let Some(expanded) = self.emit_attribute_macro_expansion(node) {
-                // we wrap it in a dummy MacroCall to get a single Item label that can replace
-                // the original Item
-                let label = self.trap.emit(generated::MacroCall {
-                    id: TrapId::Star,
-                    attrs: vec![],
-                    path: None,
-                    token_tree: None,
-                });
-                generated::MacroCall::emit_macro_call_expansion(
-                    label,
-                    expanded.into(),
-                    &mut self.trap.writer,
-                );
-                return Some(label);
-            }
-        }
-        if self.is_attribute_macro_target(node) {
-            self.macro_context_depth += 1;
-        }
-        None
-    }
-
     fn process_item_macro_expansion(
         &mut self,
         node: &impl ast::AstNode,
@@ -915,10 +884,6 @@ impl<'a> Translator<'a> {
         &mut self,
         node: &ast::Item,
     ) -> Option<Label<generated::MacroItems>> {
-        if !self.is_attribute_macro_target(node) {
-            return None;
-        }
-        self.macro_context_depth -= 1;
         if self.macro_context_depth > 0 {
             // only expand the outermost attribute macro
             return None;
@@ -927,7 +892,48 @@ impl<'a> Translator<'a> {
         self.process_item_macro_expansion(node, expansion.map(|x| x.value))
     }
 
-    pub(crate) fn emit_item_expansion(&mut self, node: &ast::Item, label: Label<generated::Item>) {
+    pub(crate) fn item_pre_emit(
+        &mut self,
+        node: &ast::Item,
+    ) -> Option<Label<generated::MacroCall>> {
+        if !self.is_attribute_macro_target(node) {
+            return None;
+        }
+        if self.source_kind == SourceKind::Library {
+            // if the item expands via an attribute macro, we want to only emit the expansion
+            if let Some(expanded) = self.emit_attribute_macro_expansion(node) {
+                // we wrap it in a dummy MacroCall to get a single Item label that can replace
+                // the original Item
+                let label = self.trap.emit(generated::MacroCall {
+                    id: TrapId::Star,
+                    attrs: vec![],
+                    path: None,
+                    token_tree: None,
+                });
+                generated::MacroCall::emit_macro_call_expansion(
+                    label,
+                    expanded.into(),
+                    &mut self.trap.writer,
+                );
+                return Some(label);
+            }
+        }
+        self.macro_context_depth += 1;
+        None
+    }
+
+    pub(crate) fn item_post_emit(&mut self, node: &ast::Item, label: Label<generated::Item>) {
+        if !self.is_attribute_macro_target(node) {
+            return;
+        }
+        // see `item_pre_emit`:
+        // if self.is_attribute_macro_target(node), then we either exited early with `Some(label)`
+        // and are not here, or we did self.macro_context_depth += 1
+        assert!(
+            self.macro_context_depth > 0,
+            "macro_context_depth should be > 0 for an attribute macro target"
+        );
+        self.macro_context_depth -= 1;
         if let Some(expanded) = self.emit_attribute_macro_expansion(node) {
             generated::Item::emit_attribute_macro_expansion(label, expanded, &mut self.trap.writer);
         }