From 885e89a927e15b21f6b62e3fb4608fd041fc9a10 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 30 Aug 2024 17:45:51 +0200 Subject: [PATCH] Rust: first running tests --- rust/.generated.list | 6 +- rust/.gitattributes | 6 +- rust/codegen/README.md | 6 - rust/codeql-extractor.yml | 1 + rust/extractor/src/config.rs | 18 ++- rust/extractor/src/main.rs | 5 +- rust/ql/lib/codeql/rust/elements/Element.qll | 5 +- rust/ql/lib/codeql/rust/elements/File.qll | 107 ++++++++++++++- rust/ql/lib/codeql/rust/elements/Function.qll | 5 +- .../ql/lib/codeql/rust/elements/Locatable.qll | 18 ++- rust/ql/lib/codeql/rust/elements/Location.qll | 25 +++- .../lib/codeql/rust/elements/UnknownFile.qll | 5 +- .../codeql/rust/elements/UnknownLocation.qll | 17 ++- rust/ql/lib/codeql/rust/printast/PrintAst.qll | 49 +++++++ .../lib/codeql/rust/printast/PrintAstNode.qll | 129 ++++++++++++++++++ rust/ql/lib/qlpack.lock.yml | 4 + rust/ql/lib/qlpack.yml | 15 ++ rust/ql/lib/rust.qll | 3 + rust/ql/src/qlpack.lock.yml | 4 + rust/ql/src/qlpack.yml | 12 ++ .../ide-contextual-queries/printAst.ql | 33 +++++ rust/ql/test/.gitignore | 3 + rust/ql/test/TestUtils.qll | 13 ++ .../generated/File/File.expected | 1 + .../extractor-tests/generated/File/File.ql | 10 ++ .../generated/File/MISSING_SOURCE.txt | 4 - .../extractor-tests/generated/File/test.rs | 1 + .../generated/Function/Function.expected | 3 + .../generated/Function/Function.ql | 10 ++ .../generated/Function/MISSING_SOURCE.txt | 4 - .../generated/Function/test.rs | 3 + rust/ql/test/qlpack.lock.yml | 4 + rust/ql/test/qlpack.yml | 8 ++ rust/tools/BUILD.bazel | 4 +- rust/tools/index-files.sh | 5 + rust/tools/index.sh | 17 --- rust/tools/qltest.sh | 32 +++++ 37 files changed, 531 insertions(+), 64 deletions(-) delete mode 100644 rust/codegen/README.md create mode 100644 rust/ql/lib/codeql/rust/printast/PrintAst.qll create mode 100644 rust/ql/lib/codeql/rust/printast/PrintAstNode.qll create mode 100644 rust/ql/lib/qlpack.lock.yml create mode 100644 rust/ql/lib/qlpack.yml create mode 100644 rust/ql/lib/rust.qll create mode 100644 rust/ql/src/qlpack.lock.yml create mode 100644 rust/ql/src/qlpack.yml create mode 100644 rust/ql/src/queries/ide-contextual-queries/printAst.ql create mode 100644 rust/ql/test/.gitignore create mode 100644 rust/ql/test/TestUtils.qll create mode 100644 rust/ql/test/extractor-tests/generated/File/File.expected create mode 100644 rust/ql/test/extractor-tests/generated/File/File.ql delete mode 100644 rust/ql/test/extractor-tests/generated/File/MISSING_SOURCE.txt create mode 100644 rust/ql/test/extractor-tests/generated/File/test.rs create mode 100644 rust/ql/test/extractor-tests/generated/Function/Function.expected create mode 100644 rust/ql/test/extractor-tests/generated/Function/Function.ql delete mode 100644 rust/ql/test/extractor-tests/generated/Function/MISSING_SOURCE.txt create mode 100644 rust/ql/test/extractor-tests/generated/Function/test.rs create mode 100644 rust/ql/test/qlpack.lock.yml create mode 100644 rust/ql/test/qlpack.yml create mode 100755 rust/tools/index-files.sh delete mode 100755 rust/tools/index.sh create mode 100755 rust/tools/qltest.sh diff --git a/rust/.generated.list b/rust/.generated.list index 9f13a9da75a..3b67e220812 100644 --- a/rust/.generated.list +++ b/rust/.generated.list @@ -3,8 +3,6 @@ ql/lib/codeql/rust/elements/DbFileConstructor.qll ea93dc49b23b1c6d800ab9d0b9cacf ql/lib/codeql/rust/elements/DbLocation.qll 1f694594e8e4ab65a8781cd443ad4f864447ca88e2cb65504aee5a779393c84d 003ec72275406eb8f5ddd6ccc2b258fb7c906d4bb2c0ef1ba235f291624321ca ql/lib/codeql/rust/elements/DbLocationConstructor.qll 8848abace985818a5d3a6eddfc4cb200795970146d282b037b4f22ae6230b894 44dba880e17bb1072fa12451ccaae4830fd04dcc61f7403d35510309fde6906e ql/lib/codeql/rust/elements/Declaration.qll d4ec5c83728f1837243caf2f27d06fd05ecdd2ca440112accff99bfd37b45e5f c1cd9b297be8b69207e75d24b29949b9f71c78406ee0ffd38d0b0810288d6140 -ql/lib/codeql/rust/elements/Element.qll 2bffdeb8863a9853d019e56afa268b3d39fb3687785ad636b159656c199997ca 18020582c7d1b89f4073b6deaf793c3072fdc52503a6cc53172c96282f141de4 -ql/lib/codeql/rust/elements/File.qll 75144234638417347107b75db80c561379e6372859ebc463276aef716e0ed42b 55bb468873acf24e6828faf7be3ff8963c9a4eb344bfedd95328a51b6f00a226 ql/lib/codeql/rust/elements/Function.qll 220eb1d0e6b49e83b7674a9f505d3f529cf7f4d022ddca44fa6367e0d75daa0f d156cff59ecdcffc0d62041b801025b1aaedbc92d44efd107f93f6fd06c34a6d ql/lib/codeql/rust/elements/FunctionConstructor.qll a9269b37182c0bf432f9b2b015691da5dbd64819b9bd25445af229d873014a91 69107a7503af14a51e091e6918094a4e9fc316a72de2e1514f001872ce0f2c0c ql/lib/codeql/rust/elements/Locatable.qll dfc0235cf5aff0ec91d85375ac03998b0e6a69f3722b099330ab0a2161af988a bb1055f59ef7e95e89748765c79f1b5832aa18226f227e689a6801adfc6247ad @@ -30,6 +28,6 @@ ql/lib/codeql/rust/generated/Synth.qll d278de9c2d06cb7549cd8f2e10ed186827a2ceab6 ql/lib/codeql/rust/generated/SynthConstructors.qll 35b36df0c4fff05bcbd4ed10b1e6fa2e58fe8d8c818e7805111044825788fc01 35b36df0c4fff05bcbd4ed10b1e6fa2e58fe8d8c818e7805111044825788fc01 ql/lib/codeql/rust/generated/UnknownFile.qll ec9d1a3f15ecbf1743d4e39cb3b2f217aa9b54951c93302c2c4c238c3f0ce595 ec9d1a3f15ecbf1743d4e39cb3b2f217aa9b54951c93302c2c4c238c3f0ce595 ql/lib/codeql/rust/generated/UnknownLocation.qll a19e2838c52d702d268ae530f3dbd6fcd8bb28a237a52636a960f225454103cf a19e2838c52d702d268ae530f3dbd6fcd8bb28a237a52636a960f225454103cf -ql/test/extractor-tests/generated/File/MISSING_SOURCE.txt cc7c395e7c651d62596826b1a0bedf10f35d01b8afeef47600b4ddaf804f406e cc7c395e7c651d62596826b1a0bedf10f35d01b8afeef47600b4ddaf804f406e -ql/test/extractor-tests/generated/Function/MISSING_SOURCE.txt cc7c395e7c651d62596826b1a0bedf10f35d01b8afeef47600b4ddaf804f406e cc7c395e7c651d62596826b1a0bedf10f35d01b8afeef47600b4ddaf804f406e +ql/test/extractor-tests/generated/File/File.ql dec43be882fad904fab0c6447ca93633d801cb08ff8bec309befde7d2b9e5dda 74e1f1d698558c35fa03935cc34f4c8145d376b56d7657b18aeb338f5ca752cf +ql/test/extractor-tests/generated/Function/Function.ql ae5d44a85047d50d8fbd3b62290c6935f061f07076b0070998173957e54eb43f 3e7fb6fb82463b96577394213915d8deae5332acdec2fcc07aa3eb8560420edd ql/test/extractor-tests/generated/Module/MISSING_SOURCE.txt cc7c395e7c651d62596826b1a0bedf10f35d01b8afeef47600b4ddaf804f406e cc7c395e7c651d62596826b1a0bedf10f35d01b8afeef47600b4ddaf804f406e diff --git a/rust/.gitattributes b/rust/.gitattributes index ebd94575437..b7f99adb7d5 100644 --- a/rust/.gitattributes +++ b/rust/.gitattributes @@ -5,8 +5,6 @@ /ql/lib/codeql/rust/elements/DbLocation.qll linguist-generated /ql/lib/codeql/rust/elements/DbLocationConstructor.qll linguist-generated /ql/lib/codeql/rust/elements/Declaration.qll linguist-generated -/ql/lib/codeql/rust/elements/Element.qll linguist-generated -/ql/lib/codeql/rust/elements/File.qll linguist-generated /ql/lib/codeql/rust/elements/Function.qll linguist-generated /ql/lib/codeql/rust/elements/FunctionConstructor.qll linguist-generated /ql/lib/codeql/rust/elements/Locatable.qll linguist-generated @@ -32,6 +30,6 @@ /ql/lib/codeql/rust/generated/SynthConstructors.qll linguist-generated /ql/lib/codeql/rust/generated/UnknownFile.qll linguist-generated /ql/lib/codeql/rust/generated/UnknownLocation.qll linguist-generated -/ql/test/extractor-tests/generated/File/MISSING_SOURCE.txt linguist-generated -/ql/test/extractor-tests/generated/Function/MISSING_SOURCE.txt linguist-generated +/ql/test/extractor-tests/generated/File/File.ql linguist-generated +/ql/test/extractor-tests/generated/Function/Function.ql linguist-generated /ql/test/extractor-tests/generated/Module/MISSING_SOURCE.txt linguist-generated diff --git a/rust/codegen/README.md b/rust/codegen/README.md deleted file mode 100644 index 66a811b5f41..00000000000 --- a/rust/codegen/README.md +++ /dev/null @@ -1,6 +0,0 @@ -This package aliases [`misc/codegen`](../misc/codegen) providing the Rust-specific options -in [`rust/codegen.conf`](../codegen.conf). - -Running `bazel run //rust/codegen` will generate all checked in -files ([dbscheme](../ql/lib/swift.dbscheme), [QL generated code](../ql/lib/codeql/swift/generated), -[generated QL stubs](../ql/lib/codeql/swift/elements), [generated QL tests](../ql/test/extractor-tests/generated)). diff --git a/rust/codeql-extractor.yml b/rust/codeql-extractor.yml index af7d4475867..257fd8e7c7f 100644 --- a/rust/codeql-extractor.yml +++ b/rust/codeql-extractor.yml @@ -2,6 +2,7 @@ name: "rust" display_name: "Rust" version: 0.1.0 column_kind: "utf8" +legacy_qltest_extraction: true build_modes: - none github_api_languages: diff --git a/rust/extractor/src/config.rs b/rust/extractor/src/config.rs index 521718c831a..08db09f8744 100644 --- a/rust/extractor/src/config.rs +++ b/rust/extractor/src/config.rs @@ -1,4 +1,5 @@ use std::path::PathBuf; +use anyhow::Context; use serde::{Deserialize, Serialize, Serializer, Deserializer}; use serde_with; use figment::{Figment, providers::{Env, Serialized}}; @@ -48,7 +49,9 @@ struct CliArgs { #[arg(long)] compression: Option, #[arg(short, long, action = ArgAction::Count)] - pub verbose: u8, + verbose: u8, + #[arg(long)] + inputs_file: Option, inputs: Vec, } @@ -58,10 +61,15 @@ fn is_default(t: &T) -> bool { } impl Config { - pub fn extract() -> figment::Result { - Figment::new() + pub fn extract() -> anyhow::Result { + let mut cli_args = CliArgs::parse(); + if let Some(inputs_file) = cli_args.inputs_file.take() { + let inputs_list = std::fs::read_to_string(inputs_file).context("reading file list")?; + cli_args.inputs.extend(inputs_list.split("\n").map(PathBuf::from)); + } + Ok(Figment::new() .merge(Env::prefixed("CODEQL_EXTRACTOR_RUST_")) - .merge(Serialized::defaults(CliArgs::parse())) - .extract() + .merge(Serialized::defaults(cli_args)) + .extract().context("loading configuration")?) } } diff --git a/rust/extractor/src/main.rs b/rust/extractor/src/main.rs index ff98d58c314..1044faafd95 100644 --- a/rust/extractor/src/main.rs +++ b/rust/extractor/src/main.rs @@ -35,8 +35,7 @@ fn main() -> anyhow::Result<()> { prefill_caches: false, }; for input in cfg.inputs { - load_workspace_at(&input, &config, &load_config, &no_progress)?; - let (db, vfs, _macro_server) = load_workspace_at(&input, &config, &load_config, &no_progress)?; + let (db, vfs, _macro_server) = load_workspace_at(&input, &config, &load_config, &no_progress).context("loading inputs")?; let crates = ::crate_graph(&db); for crate_id in crates.iter().take(1) { let krate = Crate::from(crate_id); @@ -49,7 +48,7 @@ fn main() -> anyhow::Result<()> { &krate, &vfs, &archiver, - ).emit_crate()?; + ).emit_crate().context("writing trap file")?; } } Ok(()) diff --git a/rust/ql/lib/codeql/rust/elements/Element.qll b/rust/ql/lib/codeql/rust/elements/Element.qll index bfdfcad58ae..c11ea4d23d7 100644 --- a/rust/ql/lib/codeql/rust/elements/Element.qll +++ b/rust/ql/lib/codeql/rust/elements/Element.qll @@ -1,8 +1,9 @@ -// generated by codegen, remove this comment if you wish to edit this file /** * This module provides a hand-modifiable wrapper around the generated class `Element`. */ private import codeql.rust.generated.Element -class Element extends Generated::Element { } +class Element extends Generated::Element { + predicate isUnknown() { none() } // compatibility with test generation, to be fixed +} diff --git a/rust/ql/lib/codeql/rust/elements/File.qll b/rust/ql/lib/codeql/rust/elements/File.qll index eb7dd68c051..f9d2e62017f 100644 --- a/rust/ql/lib/codeql/rust/elements/File.qll +++ b/rust/ql/lib/codeql/rust/elements/File.qll @@ -1,8 +1,111 @@ -// generated by codegen, remove this comment if you wish to edit this file /** * This module provides a hand-modifiable wrapper around the generated class `File`. */ private import codeql.rust.generated.File +private import codeql.rust.elements.Location +private import codeql.rust.elements.UnknownLocation -class File extends Generated::File { } +class File extends Generated::File { + /** toString */ + override string toString() { result = this.getAbsolutePath() } + + /** Gets the absolute path of this file. */ + string getAbsolutePath() { result = this.getName() } + + /** Gets the full name of this file. */ + string getFullName() { result = this.getAbsolutePath() } + + /** Gets the URL of this file. */ + string getURL() { result = "file://" + this.getAbsolutePath() + ":0:0:0:0" } + + /** + * Holds if either, + * - `part` is the base name of this container and `i = 1`, or + * - `part` is the stem of this container and `i = 2`, or + * - `part` is the extension of this container and `i = 3`. + */ + cached + private predicate splitAbsolutePath(string part, int i) { + part = this.getAbsolutePath().regexpCapture(".*/(([^/]*?)(?:\\.([^.]*))?)", i) + } + + /** Gets the base name of this file. */ + string getBaseName() { this.splitAbsolutePath(result, 1) } + + /** + * Gets the extension of this container, that is, the suffix of its base name + * after the last dot character, if any. + * + * In particular, + * + * - if the name does not include a dot, there is no extension, so this + * predicate has no result; + * - if the name ends in a dot, the extension is the empty string; + * - if the name contains multiple dots, the extension follows the last dot. + * + * Here are some examples of absolute paths and the corresponding extensions + * (surrounded with quotes to avoid ambiguity): + * + * + * + * + * + * + * + * + *
Absolute pathExtension
"/tmp/tst.txt""txt"
"/tmp/.classpath""classpath"
"/bin/bash"not defined
"/tmp/tst2."""
"/tmp/x.tar.gz""gz"
+ */ + string getExtension() { this.splitAbsolutePath(result, 3) } + + /** + * Gets the stem of this container, that is, the prefix of its base name up to + * (but not including) the last dot character if there is one, or the entire + * base name if there is not. + * + * Here are some examples of absolute paths and the corresponding stems + * (surrounded with quotes to avoid ambiguity): + * + * + * + * + * + * + * + * + *
Absolute pathStem
"/tmp/tst.txt""tst"
"/tmp/.classpath"""
"/bin/bash""bash"
"/tmp/tst2.""tst2"
"/tmp/x.tar.gz""x.tar"
+ */ + string getStem() { this.splitAbsolutePath(result, 2) } + + /** + * Gets the number of lines containing code in this file. This value + * is approximate. + */ + int getNumberOfLinesOfCode() { + result = + count(int line | + exists(Location loc | + not loc instanceof UnknownLocation and loc.getFile() = this and loc.getStartLine() = line + ) + ) + } + + /** + * Gets the relative path of this file from the root folder of the + * analyzed source location. The relative path of the root folder itself + * would be the empty string. + * + * This has no result if the file is outside the source root, that is, + * if the root folder is not a reflexive, transitive parent of this file. + */ + string getRelativePath() { + exists(string absPath, string pref | + absPath = this.getAbsolutePath() and sourceLocationPrefix(pref) + | + absPath = pref and result = "" + or + absPath = pref.regexpReplaceAll("/$", "") + "/" + result and + not result.matches("/%") + ) + } +} diff --git a/rust/ql/lib/codeql/rust/elements/Function.qll b/rust/ql/lib/codeql/rust/elements/Function.qll index 0b6e85fccff..6ede5224009 100644 --- a/rust/ql/lib/codeql/rust/elements/Function.qll +++ b/rust/ql/lib/codeql/rust/elements/Function.qll @@ -1,8 +1,9 @@ -// generated by codegen, remove this comment if you wish to edit this file /** * This module provides a hand-modifiable wrapper around the generated class `Function`. */ private import codeql.rust.generated.Function -class Function extends Generated::Function { } +class Function extends Generated::Function { + override string toString() { result = this.getName() } +} diff --git a/rust/ql/lib/codeql/rust/elements/Locatable.qll b/rust/ql/lib/codeql/rust/elements/Locatable.qll index ce2128d0ea6..f6df70bb089 100644 --- a/rust/ql/lib/codeql/rust/elements/Locatable.qll +++ b/rust/ql/lib/codeql/rust/elements/Locatable.qll @@ -1,8 +1,22 @@ -// generated by codegen, remove this comment if you wish to edit this file /** * This module provides a hand-modifiable wrapper around the generated class `Locatable`. */ private import codeql.rust.generated.Locatable +private import codeql.rust.elements.File +private import codeql.rust.elements.UnknownLocation -class Locatable extends Generated::Locatable { } +class Locatable extends Generated::Locatable { + pragma[nomagic] + override Location getLocation() { + result = Generated::Locatable.super.getLocation() + or + not exists(Generated::Locatable.super.getLocation()) and + result instanceof UnknownLocation + } + + /** + * Gets the primary file where this element occurs. + */ + File getFile() { result = this.getLocation().getFile() } +} diff --git a/rust/ql/lib/codeql/rust/elements/Location.qll b/rust/ql/lib/codeql/rust/elements/Location.qll index 420e91a1c8d..a433422337d 100644 --- a/rust/ql/lib/codeql/rust/elements/Location.qll +++ b/rust/ql/lib/codeql/rust/elements/Location.qll @@ -1,8 +1,29 @@ -// generated by codegen, remove this comment if you wish to edit this file /** * This module provides a hand-modifiable wrapper around the generated class `Location`. */ private import codeql.rust.generated.Location -class Location extends Generated::Location { } +class Location extends Generated::Location { + /** + * Holds if this location is described by `path`, `startLine`, `startColumn`, `endLine` and `endColumn`. + */ + predicate hasLocationInfo(string path, int startLine, int startColumn, int endLine, int endColumn) { + path = this.getFile().getFullName() and + startLine = this.getStartLine() and + startColumn = this.getStartColumn() and + endLine = this.getEndLine() and + endColumn = this.getEndColumn() + } + + /** + * Gets a textual representation of this location. + */ + override string toString() { + exists(string filePath, int startLine, int startColumn, int endLine, int endColumn | + this.hasLocationInfo(filePath, startLine, startColumn, endLine, endColumn) + | + toUrl(filePath, startLine, startColumn, endLine, endColumn, result) + ) + } +} diff --git a/rust/ql/lib/codeql/rust/elements/UnknownFile.qll b/rust/ql/lib/codeql/rust/elements/UnknownFile.qll index 8378e9b21dd..30c2ab8ba1c 100644 --- a/rust/ql/lib/codeql/rust/elements/UnknownFile.qll +++ b/rust/ql/lib/codeql/rust/elements/UnknownFile.qll @@ -1,8 +1,9 @@ -// generated by codegen, remove this comment if you wish to edit this file /** * This module provides a hand-modifiable wrapper around the generated class `UnknownFile`. */ private import codeql.rust.generated.UnknownFile -class UnknownFile extends Generated::UnknownFile { } +class UnknownFile extends Generated::UnknownFile { + override string getName() { result = "" } +} diff --git a/rust/ql/lib/codeql/rust/elements/UnknownLocation.qll b/rust/ql/lib/codeql/rust/elements/UnknownLocation.qll index 64cc97dc9b6..0374aa833b8 100644 --- a/rust/ql/lib/codeql/rust/elements/UnknownLocation.qll +++ b/rust/ql/lib/codeql/rust/elements/UnknownLocation.qll @@ -1,8 +1,21 @@ -// generated by codegen, remove this comment if you wish to edit this file /** * This module provides a hand-modifiable wrapper around the generated class `UnknownLocation`. */ private import codeql.rust.generated.UnknownLocation +private import codeql.rust.elements.File +private import codeql.rust.elements.UnknownFile -class UnknownLocation extends Generated::UnknownLocation { } +class UnknownLocation extends Generated::UnknownLocation { + override File getFile() { result instanceof UnknownFile } + + override int getStartLine() { result = 0 } + + override int getStartColumn() { result = 0 } + + override int getEndLine() { result = 0 } + + override int getEndColumn() { result = 0 } + + override string toString() { result = "UnknownLocation" } +} diff --git a/rust/ql/lib/codeql/rust/printast/PrintAst.qll b/rust/ql/lib/codeql/rust/printast/PrintAst.qll new file mode 100644 index 00000000000..4527058adc8 --- /dev/null +++ b/rust/ql/lib/codeql/rust/printast/PrintAst.qll @@ -0,0 +1,49 @@ +/** + * Provides queries to pretty-print a Swift AST as a graph. + */ + +import PrintAstNode + +cached +private int getOrder(PrintAstNode node) { + node = + rank[result](PrintAstNode n, Location loc | + loc = n.getLocation() + | + n + order by + loc.getFile().getName(), loc.getStartLine(), loc.getStartColumn(), loc.getEndLine(), + loc.getEndColumn() + ) +} + +/** Holds if `node` belongs to the output tree, and its property `key` has the given `value`. */ +query predicate nodes(PrintAstNode node, string key, string value) { + node.shouldBePrinted() and + ( + key = "semmle.label" and value = node.toString() + or + key = "semmle.order" and value = getOrder(node).toString() + or + value = node.getProperty(key) + ) +} + +/** + * Holds if `target` is a child of `source` in the AST, and property `key` of the edge has the + * given `value`. + */ +query predicate edges(PrintAstNode source, PrintAstNode target, string key, string value) { + source.shouldBePrinted() and + target.shouldBePrinted() and + exists(int index, string accessor | source.hasChild(target, index, accessor) | + key = "semmle.label" and value = accessor + or + key = "semmle.order" and value = index.toString() + ) +} + +/** Holds if property `key` of the graph has the given `value`. */ +query predicate graphProperties(string key, string value) { + key = "semmle.graphKind" and value = "tree" +} diff --git a/rust/ql/lib/codeql/rust/printast/PrintAstNode.qll b/rust/ql/lib/codeql/rust/printast/PrintAstNode.qll new file mode 100644 index 00000000000..e52342d619f --- /dev/null +++ b/rust/ql/lib/codeql/rust/printast/PrintAstNode.qll @@ -0,0 +1,129 @@ +/** + * Provides classes used to pretty-print a Swift AST as a graph. + * This is factored out of `PrintAst.qll` for testing purposes. + */ + +import rust +import codeql.rust.generated.ParentChild + +private newtype TPrintAstConfiguration = TMakePrintAstConfiguration() + +/** + * The hook to customize the files and functions printed by this module. + */ +class PrintAstConfiguration extends TPrintAstConfiguration { + /** + * Gets the string representation of this singleton + */ + string toString() { result = "PrintAstConfiguration" } + + /** + * Holds if the AST for `e` should be printed. By default, holds for all. + */ + predicate shouldPrint(Locatable e) { not e instanceof Diagnostics and not e instanceof MacroRole } +} + +private predicate shouldPrint(Locatable e) { any(PrintAstConfiguration config).shouldPrint(e) } + +/** + * An AST node that should be printed. + */ +private newtype TPrintAstNode = TPrintLocatable(Locatable ast) + +/** + * A node in the output tree. + */ +class PrintAstNode extends TPrintAstNode { + /** + * Gets a textual representation of this node. + */ + abstract string toString(); + + /** + * Gets the child node at index `index`. Child indices must be unique, + * but need not be contiguous. + */ + abstract predicate hasChild(PrintAstNode child, int index, string label); + + /** + * Holds if this node should be printed in the output. + */ + abstract predicate shouldBePrinted(); + + /** + * Gets the location of this node in the source code. + */ + abstract Location getLocation(); + + /** + * Gets the value of an additional property of this node, where the name of + * the property is `key`. + */ + string getProperty(string key) { none() } + + /** + * Gets the underlying AST node, if any. + */ + abstract Locatable getAstNode(); +} + +private string prettyPrint(Locatable e) { + result = "[" + concat(e.getPrimaryQlClasses(), ", ") + "] " + e +} + +private class Unresolved extends Locatable { + Unresolved() { this != this.resolve() } +} + +/** + * A graph node representing a real Locatable node. + */ +class PrintLocatable extends PrintAstNode, TPrintLocatable { + Locatable ast; + + PrintLocatable() { this = TPrintLocatable(ast) } + + override string toString() { result = prettyPrint(ast) } + + final override predicate shouldBePrinted() { shouldPrint(ast) } + + override predicate hasChild(PrintAstNode child, int index, string label) { + exists(Locatable c, int i, string accessor | + c = getChildAndAccessor(ast, i, accessor) and + ( + // use even indexes for normal children, leaving odd slots for conversions if any + child = TPrintLocatable(c) and index = 2 * i and label = accessor + or + child = TPrintLocatable(c.getFullyUnresolved().(Unresolved)) and + index = 2 * i + 1 and + ( + if c instanceof Expr + then label = accessor + ".getFullyConverted()" + else label = accessor + ".getFullyUnresolved()" + ) + ) + ) + } + + final override Locatable getAstNode() { result = ast } + + final override Location getLocation() { result = ast.getLocation() } +} + +/** + * A specialization of graph node for "unresolved" children, that is nodes in + * the parallel conversion AST. + */ +class PrintUnresolved extends PrintLocatable { + override Unresolved ast; + + override predicate hasChild(PrintAstNode child, int index, string label) { + // only print immediate unresolved children from the "parallel" AST + child = TPrintLocatable(getImmediateChildAndAccessor(ast, index, label).(Unresolved)) + } +} + +private predicate hasPropertyWrapperElement(VarDecl d, Locatable a) { + a = [d.getPropertyWrapperBackingVar(), d.getPropertyWrapperProjectionVar()] or + a = [d.getPropertyWrapperBackingVarBinding(), d.getPropertyWrapperProjectionVarBinding()] +} diff --git a/rust/ql/lib/qlpack.lock.yml b/rust/ql/lib/qlpack.lock.yml new file mode 100644 index 00000000000..06dd07fc7dc --- /dev/null +++ b/rust/ql/lib/qlpack.lock.yml @@ -0,0 +1,4 @@ +--- +dependencies: {} +compiled: false +lockVersion: 1.0.0 diff --git a/rust/ql/lib/qlpack.yml b/rust/ql/lib/qlpack.yml new file mode 100644 index 00000000000..649d02c8e3a --- /dev/null +++ b/rust/ql/lib/qlpack.yml @@ -0,0 +1,15 @@ +name: codeql/rust-all +version: 0.1.0-dev +groups: rust +extractor: rust +dbscheme: rust.dbscheme +library: true +dependencies: + codeql/controlflow: ${workspace} + codeql/dataflow: ${workspace} + codeql/regex: ${workspace} + codeql/mad: ${workspace} + codeql/ssa: ${workspace} + codeql/tutorial: ${workspace} + codeql/util: ${workspace} +warnOnImplicitThis: true diff --git a/rust/ql/lib/rust.qll b/rust/ql/lib/rust.qll new file mode 100644 index 00000000000..cedced159c3 --- /dev/null +++ b/rust/ql/lib/rust.qll @@ -0,0 +1,3 @@ +/** Top-level import for the Rust language pack */ + +import codeql.rust.elements diff --git a/rust/ql/src/qlpack.lock.yml b/rust/ql/src/qlpack.lock.yml new file mode 100644 index 00000000000..06dd07fc7dc --- /dev/null +++ b/rust/ql/src/qlpack.lock.yml @@ -0,0 +1,4 @@ +--- +dependencies: {} +compiled: false +lockVersion: 1.0.0 diff --git a/rust/ql/src/qlpack.yml b/rust/ql/src/qlpack.yml new file mode 100644 index 00000000000..1638da417ce --- /dev/null +++ b/rust/ql/src/qlpack.yml @@ -0,0 +1,12 @@ +name: codeql/rust-queries +version: 0.1.0-dev +groups: + - rust + - queries +suites: codeql-suites +extractor: rust +dependencies: + codeql/rust-all: ${workspace} + codeql/suite-helpers: ${workspace} + codeql/util: ${workspace} +warnOnImplicitThis: true diff --git a/rust/ql/src/queries/ide-contextual-queries/printAst.ql b/rust/ql/src/queries/ide-contextual-queries/printAst.ql new file mode 100644 index 00000000000..2fd65ca49ab --- /dev/null +++ b/rust/ql/src/queries/ide-contextual-queries/printAst.ql @@ -0,0 +1,33 @@ +/** + * @name Print AST + * @description Outputs a representation of a file's Abstract Syntax Tree. This + * query is used by the VS Code extension. + * @id rust/print-ast + * @kind graph + * @tags ide-contextual-queries/print-ast + */ + +import rust +import codeql.rust.printast.PrintAst +import codeql.IDEContextual +import codeql.rust.generated.ParentChild + +/** + * Gets the source file to generate an AST from. + */ +external string selectedSourceFile(); + +class PrintAstConfigurationOverride extends PrintAstConfiguration { + /** + * Holds if the location matches the selected file in the VS Code extension and + * the element is `e`. + */ + override predicate shouldPrint(Locatable e) { + super.shouldPrint(e) and + ( + e.getFile() = getFileBySourceArchiveName(selectedSourceFile()) + or + exists(Locatable parent | this.shouldPrint(parent) and parent = getImmediateParent(e)) + ) + } +} diff --git a/rust/ql/test/.gitignore b/rust/ql/test/.gitignore new file mode 100644 index 00000000000..b4887cab780 --- /dev/null +++ b/rust/ql/test/.gitignore @@ -0,0 +1,3 @@ +Cargo.toml +Cargo.lock +target/ diff --git a/rust/ql/test/TestUtils.qll b/rust/ql/test/TestUtils.qll new file mode 100644 index 00000000000..317b71f2bdf --- /dev/null +++ b/rust/ql/test/TestUtils.qll @@ -0,0 +1,13 @@ +private import codeql.rust.elements + +cached +predicate toBeTested(Element e) { + exists(File f | + f.getName().matches("%rust/ql/test%") and + ( + e = f + or + e.(Locatable).getLocation().getFile() = f + ) + ) +} diff --git a/rust/ql/test/extractor-tests/generated/File/File.expected b/rust/ql/test/extractor-tests/generated/File/File.expected new file mode 100644 index 00000000000..7d8a15282a8 --- /dev/null +++ b/rust/ql/test/extractor-tests/generated/File/File.expected @@ -0,0 +1 @@ +| test.rs:0:0:0:0 | test.rs | DbFile | getName: | test.rs | diff --git a/rust/ql/test/extractor-tests/generated/File/File.ql b/rust/ql/test/extractor-tests/generated/File/File.ql new file mode 100644 index 00000000000..20550a0d1b5 --- /dev/null +++ b/rust/ql/test/extractor-tests/generated/File/File.ql @@ -0,0 +1,10 @@ +// generated by codegen +import codeql.rust.elements +import TestUtils + +from File x, string getName +where + toBeTested(x) and + not x.isUnknown() and + getName = x.getName() +select x, x.getPrimaryQlClasses(), "getName:", getName diff --git a/rust/ql/test/extractor-tests/generated/File/MISSING_SOURCE.txt b/rust/ql/test/extractor-tests/generated/File/MISSING_SOURCE.txt deleted file mode 100644 index 9cb54ddd059..00000000000 --- a/rust/ql/test/extractor-tests/generated/File/MISSING_SOURCE.txt +++ /dev/null @@ -1,4 +0,0 @@ -// generated by codegen - -After a source file is added in this directory and codegen is run again, test queries -will appear and this file will be deleted diff --git a/rust/ql/test/extractor-tests/generated/File/test.rs b/rust/ql/test/extractor-tests/generated/File/test.rs new file mode 100644 index 00000000000..f328e4d9d04 --- /dev/null +++ b/rust/ql/test/extractor-tests/generated/File/test.rs @@ -0,0 +1 @@ +fn main() {} diff --git a/rust/ql/test/extractor-tests/generated/Function/Function.expected b/rust/ql/test/extractor-tests/generated/Function/Function.expected new file mode 100644 index 00000000000..89a036410f0 --- /dev/null +++ b/rust/ql/test/extractor-tests/generated/Function/Function.expected @@ -0,0 +1,3 @@ +| test.rs:0:0:0:11 | foo | getName: | foo | +| test.rs:1:0:1:11 | bar | getName: | bar | +| test.rs:2:0:2:11 | baz | getName: | baz | diff --git a/rust/ql/test/extractor-tests/generated/Function/Function.ql b/rust/ql/test/extractor-tests/generated/Function/Function.ql new file mode 100644 index 00000000000..78201284e1f --- /dev/null +++ b/rust/ql/test/extractor-tests/generated/Function/Function.ql @@ -0,0 +1,10 @@ +// generated by codegen +import codeql.rust.elements +import TestUtils + +from Function x, string getName +where + toBeTested(x) and + not x.isUnknown() and + getName = x.getName() +select x, "getName:", getName diff --git a/rust/ql/test/extractor-tests/generated/Function/MISSING_SOURCE.txt b/rust/ql/test/extractor-tests/generated/Function/MISSING_SOURCE.txt deleted file mode 100644 index 9cb54ddd059..00000000000 --- a/rust/ql/test/extractor-tests/generated/Function/MISSING_SOURCE.txt +++ /dev/null @@ -1,4 +0,0 @@ -// generated by codegen - -After a source file is added in this directory and codegen is run again, test queries -will appear and this file will be deleted diff --git a/rust/ql/test/extractor-tests/generated/Function/test.rs b/rust/ql/test/extractor-tests/generated/Function/test.rs new file mode 100644 index 00000000000..35c72b4596e --- /dev/null +++ b/rust/ql/test/extractor-tests/generated/Function/test.rs @@ -0,0 +1,3 @@ +fn foo() {} +fn bar() {} +fn baz() {} diff --git a/rust/ql/test/qlpack.lock.yml b/rust/ql/test/qlpack.lock.yml new file mode 100644 index 00000000000..06dd07fc7dc --- /dev/null +++ b/rust/ql/test/qlpack.lock.yml @@ -0,0 +1,4 @@ +--- +dependencies: {} +compiled: false +lockVersion: 1.0.0 diff --git a/rust/ql/test/qlpack.yml b/rust/ql/test/qlpack.yml new file mode 100644 index 00000000000..7428ca33bed --- /dev/null +++ b/rust/ql/test/qlpack.yml @@ -0,0 +1,8 @@ +name: codeql/rust-tests +groups: [rust, test] +dependencies: + codeql/rust-queries: ${workspace} + codeql/rust-all: ${workspace} +extractor: rust +tests: . +warnOnImplicitThis: true diff --git a/rust/tools/BUILD.bazel b/rust/tools/BUILD.bazel index 27bc5bbe600..a61ab84edbf 100644 --- a/rust/tools/BUILD.bazel +++ b/rust/tools/BUILD.bazel @@ -2,8 +2,6 @@ load("//misc/bazel:pkg.bzl", "codeql_pkg_files") codeql_pkg_files( name = "tools", - exes = [ - "index.sh", - ], + exes = glob(["*.sh"]), visibility = ["//rust:__pkg__"], ) diff --git a/rust/tools/index-files.sh b/rust/tools/index-files.sh new file mode 100755 index 00000000000..da4b841b692 --- /dev/null +++ b/rust/tools/index-files.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +set -eu + +exec "$CODEQL_EXTRACTOR_RUST_ROOT/tools/$CODEQL_PLATFORM/extractor" --inputs-file="$1" diff --git a/rust/tools/index.sh b/rust/tools/index.sh deleted file mode 100755 index 7035f09d0a4..00000000000 --- a/rust/tools/index.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -# TODO move this to rust code - -inputs=($(find -name Cargo.toml)) - -if [ "${#inputs}" -eq 0 ]; then - inputs=($(find -name rust-project.json)) - if [ "${#inputs}" -eq 0 ]; then - inputs=($(find -name '*.rs')) - if [ "${#inputs}" -eq 0 ]; then - echo "no source files found" >&2 - exit 1 - fi - fi -fi - -exec "$CODEQL_EXTRACTOR_RUST_ROOT/tools/$CODEQL_PLATFORM/extractor" "${inputs[@]}" diff --git a/rust/tools/qltest.sh b/rust/tools/qltest.sh new file mode 100755 index 00000000000..d50b34573a8 --- /dev/null +++ b/rust/tools/qltest.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +mkdir -p "$CODEQL_EXTRACTOR_RUST_TRAP_DIR" + +export RUST_BACKTRACE=full + +QLTEST_LOG="$CODEQL_EXTRACTOR_RUST_LOG_DIR"/qltest.log + +EXTRACTOR="$CODEQL_EXTRACTOR_RUST_ROOT/tools/$CODEQL_PLATFORM/extractor" +for src in *.rs; do + echo -e "[package]\nname = \"test\"\n[lib]\npath=\"$src\"\n" > Cargo.toml + env=() + opts=("$src") + opts+=($(sed -n '1 s=//codeql-extractor-options:==p' $src)) + expected_status=$(sed -n 's=//codeql-extractor-expected-status:[[:space:]]*==p' $src) + expected_status=${expected_status:-0} + env+=($(sed -n '1 s=//codeql-extractor-env:==p' $src)) + echo >> $QLTEST_LOG + echo "env ${env[@]} $EXTRACTOR ${opts[@]}" >> "$QLTEST_LOG" + env "${env[@]}" "$EXTRACTOR" "${opts[@]}" >> $QLTEST_LOG 2>&1 + actual_status=$? + if [[ $actual_status != $expected_status ]]; then + FAILED=1 + fi +done + +rm -rf Cargo.* + +if [ -n "$FAILED" ]; then + cat "$QLTEST_LOG" # Show compiler errors on extraction failure + exit 1 +fi