hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2025-12-16 16:53:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #19945 from github/redsun82/fix-expansion-in-lib

Browse Source 
Rust: fix macro expansion in library code
This commit is contained in:
Paolo Tranquilli
2025-07-02 18:11:36 +02:00
committed by GitHub
parent 2fffa9db3c c70198e4e4
commit c10d89927d
23 changed files with 240 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
rust/extractor/src/config.rs
View File

@@ -71,6 +71,7 @@ pub struct Config {
pub proc_macro_server: Option<PathBuf>,
pub skip_path_resolution: bool,
pub extract_dependencies_as_source: bool,
pub force_library_mode: bool, // for testing purposes
}
impl Config {

13
rust/extractor/src/main.rs
View File

@@ -293,6 +293,11 @@ fn main() -> anyhow::Result<()> {
} else {
(SourceKind::Library, ResolvePaths::No)
};
let (source_mode, source_resolve_paths) = if cfg.force_library_mode {
(library_mode, library_resolve_paths)
} else {
(SourceKind::Source, resolve_paths)
};
let mut processed_files: HashSet<PathBuf, RandomState> =
HashSet::from_iter(files.iter().cloned());
for (manifest, files) in map.values().filter(|(_, files)| !files.is_empty()) {
@@ -311,12 +316,10 @@ fn main() -> anyhow::Result<()> {
file,
&semantics,
vfs,
resolve_paths,
SourceKind::Source,
source_resolve_paths,
source_mode,
),
Err(reason) => {
extractor.extract_without_semantics(file, SourceKind::Source, &reason)
}
Err(reason) => extractor.extract_without_semantics(file, source_mode, &reason),
};
}
for (file_id, file) in vfs.iter() {

2
rust/extractor/src/qltest.rs
View File

@@ -19,7 +19,7 @@ fn dump_lib() -> anyhow::Result<()> {
.iter()
.map(|p| p.file_stem().expect("results of glob must have a name"))
.filter(|&p| !["main", "lib", "proc_macro"].map(OsStr::new).contains(&p))
.map(|p| format!("mod {};", p.to_string_lossy()))
.map(|p| format!("pub mod {};", p.to_string_lossy()))
.join("\n");
fs::write("lib.rs", lib).context("writing lib.rs")
}

91
rust/extractor/src/translate/base.rs
View File

@@ -24,33 +24,31 @@ use ra_ap_syntax::{
impl Emission<ast::Item> for Translator<'_> {
fn pre_emit(&mut self, node: &ast::Item) -> Option<Label<generated::Item>> {
self.prepare_item_expansion(node).map(Into::into)
self.item_pre_emit(node).map(Into::into)
}
fn post_emit(&mut self, node: &ast::Item, label: Label<generated::Item>) {
self.emit_item_expansion(node, label);
self.item_post_emit(node, label);
}
}
impl Emission<ast::AssocItem> for Translator<'_> {
fn pre_emit(&mut self, node: &ast::AssocItem) -> Option<Label<generated::AssocItem>> {
self.prepare_item_expansion(&node.clone().into())
.map(Into::into)
self.item_pre_emit(&node.clone().into()).map(Into::into)
}
fn post_emit(&mut self, node: &ast::AssocItem, label: Label<generated::AssocItem>) {
self.emit_item_expansion(&node.clone().into(), label.into());
self.item_post_emit(&node.clone().into(), label.into());
}
}
impl Emission<ast::ExternItem> for Translator<'_> {
fn pre_emit(&mut self, node: &ast::ExternItem) -> Option<Label<generated::ExternItem>> {
self.prepare_item_expansion(&node.clone().into())
.map(Into::into)
self.item_pre_emit(&node.clone().into()).map(Into::into)
}
fn post_emit(&mut self, node: &ast::ExternItem, label: Label<generated::ExternItem>) {
self.emit_item_expansion(&node.clone().into(), label.into());
self.item_post_emit(&node.clone().into(), label.into());
}
}
@@ -849,35 +847,6 @@ impl<'a> Translator<'a> {
})
}
pub(crate) fn prepare_item_expansion(
&mut self,
node: &ast::Item,
) -> Option<Label<generated::MacroCall>> {
if self.source_kind == SourceKind::Library {
// if the item expands via an attribute macro, we want to only emit the expansion
if let Some(expanded) = self.emit_attribute_macro_expansion(node) {
// we wrap it in a dummy MacroCall to get a single Item label that can replace
// the original Item
let label = self.trap.emit(generated::MacroCall {
id: TrapId::Star,
attrs: vec![],
path: None,
token_tree: None,
});
generated::MacroCall::emit_macro_call_expansion(
label,
expanded.into(),
&mut self.trap.writer,
);
return Some(label);
}
}
if self.is_attribute_macro_target(node) {
self.macro_context_depth += 1;
}
None
}
fn process_item_macro_expansion(
&mut self,
node: &impl ast::AstNode,
@@ -915,10 +884,6 @@ impl<'a> Translator<'a> {
&mut self,
node: &ast::Item,
) -> Option<Label<generated::MacroItems>> {
if !self.is_attribute_macro_target(node) {
return None;
}
self.macro_context_depth -= 1;
if self.macro_context_depth > 0 {
// only expand the outermost attribute macro
return None;
@@ -927,7 +892,49 @@ impl<'a> Translator<'a> {
self.process_item_macro_expansion(node, expansion.map(|x| x.value))
}
pub(crate) fn emit_item_expansion(&mut self, node: &ast::Item, label: Label<generated::Item>) {
pub(crate) fn item_pre_emit(
&mut self,
node: &ast::Item,
) -> Option<Label<generated::MacroCall>> {
if !self.is_attribute_macro_target(node) {
return None;
}
if self.source_kind == SourceKind::Library {
// if the item expands via an attribute macro, we want to only emit the expansion
if let Some(expanded) = self.emit_attribute_macro_expansion(node) {
// we wrap it in a dummy MacroCall to get a single Item label that can replace
// the original Item
let label = self.trap.emit(generated::MacroCall {
id: TrapId::Star,
attrs: vec![],
path: None,
token_tree: None,
});
generated::Item::emit_attribute_macro_expansion(
label.into(),
expanded,
&mut self.trap.writer,
);
self.emit_location(label, node);
return Some(label);
}
}
self.macro_context_depth += 1;
None
}
pub(crate) fn item_post_emit(&mut self, node: &ast::Item, label: Label<generated::Item>) {
if !self.is_attribute_macro_target(node) {
return;
}
// see `item_pre_emit`:
// if self.is_attribute_macro_target(node), then we either exited early with `Some(label)`
// and are not here, or we did self.macro_context_depth += 1
assert!(
self.macro_context_depth > 0,
"macro_context_depth should be > 0 for an attribute macro target"
);
self.macro_context_depth -= 1;
if let Some(expanded) = self.emit_attribute_macro_expansion(node) {
generated::Item::emit_attribute_macro_expansion(label, expanded, &mut self.trap.writer);
}

4
rust/ql/lib/codeql/rust/elements/internal/MacroCallImpl.qll
View File

@@ -32,7 +32,9 @@ module Impl {
* ```
*/
class MacroCall extends Generated::MacroCall {
override string toStringImpl() { result = this.getPath().toAbbreviatedString() + "!..." }
override string toStringImpl() {
if this.hasPath() then result = this.getPath().toAbbreviatedString() + "!..." else result = ""
}
/** Gets an AST node whose location is inside the token tree belonging to this macro call. */
pragma[nomagic]

2
rust/ql/test/TestUtils.qll
View File

@@ -6,7 +6,7 @@ predicate toBeTested(Element e) {
(
not e instanceof Locatable
or
e.(Locatable).fromSource()
exists(e.(Locatable).getFile().getRelativePath())
)
}

8
rust/ql/test/extractor-tests/canonical_path/canonical_paths.expected
View File

@@ -2,8 +2,8 @@ canonicalPath
| anonymous.rs:3:1:32:1 | fn canonicals | test::anonymous::canonicals |
| anonymous.rs:34:1:36:1 | fn other | test::anonymous::other |
| {EXTERNAL LOCATION} | fn trim | <core::str>::trim |
| lib.rs:1:1:1:14 | mod anonymous | test::anonymous |
| lib.rs:2:1:2:12 | mod regular | test::regular |
| lib.rs:1:1:1:18 | mod anonymous | test::anonymous |
| lib.rs:2:1:2:16 | mod regular | test::regular |
| regular.rs:1:1:2:18 | struct Struct | test::regular::Struct |
| regular.rs:2:12:2:17 | fn eq | <test::regular::Struct as core::cmp::PartialEq>::eq |
| regular.rs:2:12:2:17 | impl ...::Eq for Struct::<...> { ... } | <test::regular::Struct as core::cmp::Eq> |
@@ -42,8 +42,8 @@ canonicalPaths
| anonymous.rs:26:5:31:5 | fn usage | None | None |
| anonymous.rs:34:1:36:1 | fn other | repo::test | crate::anonymous::other |
| anonymous.rs:35:5:35:23 | struct OtherStruct | None | None |
| lib.rs:1:1:1:14 | mod anonymous | repo::test | crate::anonymous |
| lib.rs:2:1:2:12 | mod regular | repo::test | crate::regular |
| lib.rs:1:1:1:18 | mod anonymous | repo::test | crate::anonymous |
| lib.rs:2:1:2:16 | mod regular | repo::test | crate::regular |
| regular.rs:1:1:2:18 | struct Struct | repo::test | crate::regular::Struct |
| regular.rs:2:12:2:17 | fn eq | repo::test | <crate::regular::Struct as crate::cmp::PartialEq>::eq |
| regular.rs:2:12:2:17 | impl ...::Eq for Struct::<...> { ... } | None | None |

8
rust/ql/test/extractor-tests/canonical_path_disabled/canonical_paths.expected
View File

@@ -2,8 +2,8 @@ canonicalPath
| anonymous.rs:6:1:35:1 | fn canonicals | test::anonymous::canonicals |
| anonymous.rs:37:1:39:1 | fn other | test::anonymous::other |
| {EXTERNAL LOCATION} | fn trim | <core::str>::trim |
| lib.rs:1:1:1:14 | mod anonymous | test::anonymous |
| lib.rs:2:1:2:12 | mod regular | test::regular |
| lib.rs:1:1:1:18 | mod anonymous | test::anonymous |
| lib.rs:2:1:2:16 | mod regular | test::regular |
| regular.rs:4:1:5:18 | struct Struct | test::regular::Struct |
| regular.rs:5:12:5:17 | fn eq | <test::regular::Struct as core::cmp::PartialEq>::eq |
| regular.rs:5:12:5:17 | impl ...::Eq for Struct::<...> { ... } | <test::regular::Struct as core::cmp::Eq> |
@@ -42,8 +42,8 @@ canonicalPaths
| anonymous.rs:29:5:34:5 | fn usage | None | None |
| anonymous.rs:37:1:39:1 | fn other | None | None |
| anonymous.rs:38:5:38:23 | struct OtherStruct | None | None |
| lib.rs:1:1:1:14 | mod anonymous | None | None |
| lib.rs:2:1:2:12 | mod regular | None | None |
| lib.rs:1:1:1:18 | mod anonymous | None | None |
| lib.rs:2:1:2:16 | mod regular | None | None |
| regular.rs:4:1:5:18 | struct Struct | None | None |
| regular.rs:5:12:5:17 | fn eq | None | None |
| regular.rs:5:12:5:17 | impl ...::Eq for Struct::<...> { ... } | None | None |

9
rust/ql/test/extractor-tests/generated/Module/Module.expected
View File

@@ -1,15 +1,15 @@
instances
| gen_module.rs:3:1:4:8 | mod foo |
| gen_module.rs:5:1:7:1 | mod bar |
| lib.rs:1:1:1:15 | mod gen_module |
| lib.rs:1:1:1:19 | mod gen_module |
getExtendedCanonicalPath
| gen_module.rs:3:1:4:8 | mod foo | crate::gen_module::foo |
| gen_module.rs:5:1:7:1 | mod bar | crate::gen_module::bar |
| lib.rs:1:1:1:15 | mod gen_module | crate::gen_module |
| lib.rs:1:1:1:19 | mod gen_module | crate::gen_module |
getCrateOrigin
| gen_module.rs:3:1:4:8 | mod foo | repo::test |
| gen_module.rs:5:1:7:1 | mod bar | repo::test |
| lib.rs:1:1:1:15 | mod gen_module | repo::test |
| lib.rs:1:1:1:19 | mod gen_module | repo::test |
getAttributeMacroExpansion
getAttr
getItemList
@@ -17,5 +17,6 @@ getItemList
getName
| gen_module.rs:3:1:4:8 | mod foo | gen_module.rs:4:5:4:7 | foo |
| gen_module.rs:5:1:7:1 | mod bar | gen_module.rs:5:5:5:7 | bar |
| lib.rs:1:1:1:15 | mod gen_module | lib.rs:1:5:1:14 | gen_module |
| lib.rs:1:1:1:19 | mod gen_module | lib.rs:1:9:1:18 | gen_module |
getVisibility
| lib.rs:1:1:1:19 | mod gen_module | lib.rs:1:1:1:3 | Visibility |

4
rust/ql/test/extractor-tests/generated/Name/Name.expected
View File

@@ -1,8 +1,8 @@
instances
| gen_name.rs:3:4:3:12 | test_name |
| gen_name.rs:7:9:7:11 | foo |
| lib.rs:1:5:1:12 | gen_name |
| lib.rs:1:9:1:16 | gen_name |
getText
| gen_name.rs:3:4:3:12 | test_name | test_name |
| gen_name.rs:7:9:7:11 | foo | foo |
| lib.rs:1:5:1:12 | gen_name | gen_name |
| lib.rs:1:9:1:16 | gen_name | gen_name |

4
rust/ql/test/extractor-tests/generated/SourceFile/SourceFile.expected
View File

@@ -1,7 +1,7 @@
instances
| gen_source_file.rs:1:1:9:2 | SourceFile |
| lib.rs:1:1:1:20 | SourceFile |
| lib.rs:1:1:1:24 | SourceFile |
getAttr
getItem
| gen_source_file.rs:1:1:9:2 | SourceFile | 0 | gen_source_file.rs:3:1:9:1 | fn test_source_file |
| lib.rs:1:1:1:20 | SourceFile | 0 | lib.rs:1:1:1:20 | mod gen_source_file |
| lib.rs:1:1:1:24 | SourceFile | 0 | lib.rs:1:1:1:24 | mod gen_source_file |

1
rust/ql/test/extractor-tests/generated/Visibility/Visibility.expected
View File

@@ -1,3 +1,4 @@
instances
| gen_visibility.rs:7:7:7:9 | Visibility |
| lib.rs:1:1:1:3 | Visibility |
getPath

2
rust/ql/test/extractor-tests/macro-expansion/PrintAst.expected
View File

@@ -146,8 +146,10 @@ lib.rs:
# 1| [SourceFile] SourceFile
# 1| getItem(0): [Module] mod call
# 1| getName(): [Name] call
# 1| getVisibility(): [Visibility] Visibility
# 2| getItem(1): [Module] mod macro_expansion
# 2| getName(): [Name] macro_expansion
# 2| getVisibility(): [Visibility] Visibility
macro_expansion.rs:
# 1| [SourceFile] SourceFile
# 1| getItem(0): [Use] use proc_macro::{...}

53
rust/ql/test/extractor-tests/macro-in-library/Cargo.lock generated Normal file
View File

@@ -0,0 +1,53 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 4
[[package]]
name = "proc-macro2"
version = "1.0.95"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778"
dependencies = [
"unicode-ident",
]
[[package]]
name = "proc_macro"
version = "0.0.1"
dependencies = [
"quote",
"syn",
]
[[package]]
name = "quote"
version = "1.0.40"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d"
dependencies = [
"proc-macro2",
]
[[package]]
name = "syn"
version = "2.0.104"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "17b6f705963418cdb9927482fa304bc562ece2fdd4f616084c50b7023b435a40"
dependencies = [
"proc-macro2",
"quote",
"unicode-ident",
]
[[package]]
name = "test"
version = "0.0.1"
dependencies = [
"proc_macro",
]
[[package]]
name = "unicode-ident"
version = "1.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512"

63
rust/ql/test/extractor-tests/macro-in-library/PrintAst.expected Normal file
View File

@@ -0,0 +1,63 @@
lib.rs:
# 1| [SourceFile] SourceFile
# 1| getItem(0): [Module] mod macro_in_library
# 1| getName(): [Name] macro_in_library
# 1| getVisibility(): [Visibility] Visibility
macro_in_library.rs:
# 1| [SourceFile] SourceFile
# 1| getItem(0): [MacroCall]
# 2| getAttributeMacroExpansion(): [MacroItems] MacroItems
# 2| getItem(0): [Function] fn foo
# 2| getParamList(): [ParamList] ParamList
# 2| getName(): [Name] foo
# 2| getVisibility(): [Visibility] Visibility
# 2| getItem(1): [Function] fn foo_new
# 2| getParamList(): [ParamList] ParamList
# 2| getName(): [Name] foo_new
# 2| getVisibility(): [Visibility] Visibility
# 4| getItem(1): [Function] fn bar
# 4| getParamList(): [ParamList] ParamList
# 4| getName(): [Name] bar
# 4| getVisibility(): [Visibility] Visibility
proc_macro.rs:
# 1| [SourceFile] SourceFile
# 1| getItem(0): [Use] use ...::TokenStream
# 1| getUseTree(): [UseTree] ...::TokenStream
# 1| getPath(): [Path] ...::TokenStream
# 1| getQualifier(): [Path] proc_macro
# 1| getSegment(): [PathSegment] proc_macro
# 1| getIdentifier(): [NameRef] proc_macro
# 1| getSegment(): [PathSegment] TokenStream
# 1| getIdentifier(): [NameRef] TokenStream
# 2| getItem(1): [Use] use ...::quote
# 2| getUseTree(): [UseTree] ...::quote
# 2| getPath(): [Path] ...::quote
# 2| getQualifier(): [Path] quote
# 2| getSegment(): [PathSegment] quote
# 2| getIdentifier(): [NameRef] quote
# 2| getSegment(): [PathSegment] quote
# 2| getIdentifier(): [NameRef] quote
# 4| getItem(2): [Function] fn add_one
# 5| getParamList(): [ParamList] ParamList
# 5| getParam(0): [Param] : TokenStream
# 5| getTypeRepr(): [PathTypeRepr] TokenStream
# 5| getPath(): [Path] TokenStream
# 5| getSegment(): [PathSegment] TokenStream
# 5| getIdentifier(): [NameRef] TokenStream
# 5| getParam(1): [Param] : TokenStream
# 5| getTypeRepr(): [PathTypeRepr] TokenStream
# 5| getPath(): [Path] TokenStream
# 5| getSegment(): [PathSegment] TokenStream
# 5| getIdentifier(): [NameRef] TokenStream
# 4| getAttr(0): [Attr] Attr
# 4| getMeta(): [Meta] Meta
# 4| getPath(): [Path] proc_macro_attribute
# 4| getSegment(): [PathSegment] proc_macro_attribute
# 4| getIdentifier(): [NameRef] proc_macro_attribute
# 5| getName(): [Name] add_one
# 5| getRetType(): [RetTypeRepr] RetTypeRepr
# 5| getTypeRepr(): [PathTypeRepr] TokenStream
# 5| getPath(): [Path] TokenStream
# 5| getSegment(): [PathSegment] TokenStream
# 5| getIdentifier(): [NameRef] TokenStream
# 5| getVisibility(): [Visibility] Visibility

1
rust/ql/test/extractor-tests/macro-in-library/PrintAst.qlref Normal file
View File

@@ -0,0 +1 @@
utils/PrintAst.ql

6
rust/ql/test/extractor-tests/macro-in-library/macro_in_library.rs Normal file
View File

@@ -0,0 +1,6 @@
#[proc_macro::add_one]
pub fn foo() {}
pub fn bar() {
foo_new();
}

1
rust/ql/test/extractor-tests/macro-in-library/options.yml Normal file
View File

@@ -0,0 +1 @@
force_library_mode: true

14
rust/ql/test/extractor-tests/macro-in-library/proc_macro.rs Normal file
View File

@@ -0,0 +1,14 @@
use proc_macro::TokenStream;
use quote::quote;
#[proc_macro_attribute]
pub fn add_one(_attr: TokenStream, item: TokenStream) -> TokenStream {
let ast = syn::parse_macro_input!(item as syn::ItemFn);
let mut new_ast = ast.clone();
new_ast.sig.ident = syn::Ident::new(&format!("{}_new", ast.sig.ident), ast.sig.ident.span());
quote! {
#ast
#new_ast
}.into()
}

4
rust/ql/test/extractor-tests/macro-in-library/test.expected Normal file
View File

@@ -0,0 +1,4 @@
macro_items
| macro_in_library.rs:2:1:2:14 | MacroItems | 0 | macro_in_library.rs:2:1:2:14 | fn foo |
| macro_in_library.rs:2:1:2:14 | MacroItems | 1 | macro_in_library.rs:2:1:2:14 | fn foo_new |
warnings

10
rust/ql/test/extractor-tests/macro-in-library/test.ql Normal file
View File

@@ -0,0 +1,10 @@
import rust
import codeql.rust.Diagnostics
query predicate macro_items(MacroItems c, int index, Item expanded) {
exists(c.getFile().getRelativePath()) and
not c.getLocation().getFile().getAbsolutePath().matches("%proc_macro.rs") and
expanded = c.getItem(index)
}
query predicate warnings(ExtractionWarning w) { any() }

7
rust/ql/test/extractor-tests/utf8/ast.expected
View File

@@ -1,6 +1,7 @@
| lib.rs:1:1:1:21 | SourceFile |
| lib.rs:1:1:1:21 | mod utf8_identifiers |
| lib.rs:1:5:1:20 | utf8_identifiers |
| lib.rs:1:1:1:3 | Visibility |
| lib.rs:1:1:1:25 | SourceFile |
| lib.rs:1:1:1:25 | mod utf8_identifiers |
| lib.rs:1:9:1:24 | utf8_identifiers |
| utf8_identifiers.rs:1:1:4:6 | fn foo |
| utf8_identifiers.rs:1:1:12:2 | SourceFile |
| utf8_identifiers.rs:1:4:1:6 | foo |

2
rust/ql/test/library-tests/path-resolution/path-resolution.expected
View File

@@ -1,5 +1,5 @@
mod
| lib.rs:1:1:1:7 | mod my |
| lib.rs:1:1:1:11 | mod my |
| main.rs:1:1:1:7 | mod my |
| main.rs:7:1:7:8 | mod my2 |
| main.rs:13:1:37:1 | mod m1 |