From 9c284b1778e93a396085c2fe07527e91784dc5e5 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 4 Jul 2025 12:22:23 +0200 Subject: [PATCH] Just: introduce scaffolding for common verbs, and apply to rust --- justfile | 2 + misc/bazel/justfile | 5 +++ misc/just/codeql-test-run.ts | 39 ++++++++++++++++++ misc/just/forward-command.ts | 64 ++++++++++++++++++++++++++++++ misc/just/forward.just | 28 +++++++++++++ misc/just/lib.just | 63 +++++++++++++++++++++++++++++ misc/just/semmle-code-stub.just | 1 + rust/justfile | 11 +++++ rust/lint.py | 52 +++++++++++++++--------- rust/ql/integration-tests/justfile | 9 +++++ rust/ql/justfile | 4 ++ rust/ql/test/justfile | 17 ++++++++ 12 files changed, 277 insertions(+), 18 deletions(-) create mode 100644 justfile create mode 100644 misc/bazel/justfile create mode 100644 misc/just/codeql-test-run.ts create mode 100644 misc/just/forward-command.ts create mode 100644 misc/just/forward.just create mode 100644 misc/just/lib.just create mode 100644 misc/just/semmle-code-stub.just create mode 100644 rust/justfile create mode 100644 rust/ql/integration-tests/justfile create mode 100644 rust/ql/justfile create mode 100644 rust/ql/test/justfile diff --git a/justfile b/justfile new file mode 100644 index 00000000000..a25d51fd9a5 --- /dev/null +++ b/justfile @@ -0,0 +1,2 @@ +import 'misc/just/lib.just' +import 'misc/just/forward.just' diff --git a/misc/bazel/justfile b/misc/bazel/justfile new file mode 100644 index 00000000000..55b2eb9dbd8 --- /dev/null +++ b/misc/bazel/justfile @@ -0,0 +1,5 @@ +import '../just/lib.just' + +[no-cd, positional-arguments, no-exit-message] +hello +ARGS: + @echo "hello from bzl" "$@" diff --git a/misc/just/codeql-test-run.ts b/misc/just/codeql-test-run.ts new file mode 100644 index 00000000000..ae68ba9dde5 --- /dev/null +++ b/misc/just/codeql-test-run.ts @@ -0,0 +1,39 @@ +import * as child_process from "child_process"; +import * as path from "path"; + +function codeqlTestRun(argv: string[]): number { + const [language, args, ...plus] = argv; + let codeql = + process.env["SEMMLE_CODE"] ? + path.join(process.env["SEMMLE_CODE"], "target", "intree", `codeql-${language}`, "codeql") + : + "codeql" + ; + process.env["CODEQL_CONFIG_FILE"] ||= "." // disable the default implicit config file, but keep an explicit one + let plus_options = plus.map(option => option.trim().split("\n").filter(option => option !== "")); + let testing_level = 0; + let parsed_args = args.split(" ").filter(arg => { + if (arg === "") return false; + if (/^\++$/.test(arg)) { + testing_level = Math.max(testing_level, arg.length); + return false; + } + return true; + }); + if (parsed_args.every(arg => arg.startsWith("-"))) { + parsed_args.push("."); + } + let invocation = [codeql, "test", "run", "-j0", ...parsed_args]; + for (let i = 0; i < Math.min(plus_options.length, testing_level); i++) { + invocation.push(...plus_options[i]); + } + console.log(`${process.env["CMD_BEGIN"] || ""}${invocation.join(" ")}${process.env["CMD_END"] || ""}`); + try { + child_process.execFileSync(invocation[0], invocation.slice(1), { stdio: "inherit" }); + } catch (error) { + return 1; + } + return 0; +} + +process.exit(codeqlTestRun(process.argv.slice(2))); diff --git a/misc/just/forward-command.ts b/misc/just/forward-command.ts new file mode 100644 index 00000000000..92078b06558 --- /dev/null +++ b/misc/just/forward-command.ts @@ -0,0 +1,64 @@ +import * as child_process from "child_process"; +import * as path from "path"; +import * as fs from "fs"; + +function commonDir(paths: string[]): string { + if (paths.length === 0) return ""; + const splitPaths = paths.map(p => p.split(path.sep)); + let i; + for (i = 0; i < splitPaths[0].length; i++) { + if (!splitPaths.every(parts => parts[i] === splitPaths[0][i])) { + break; + } + } + const commonParts = splitPaths[0].slice(0, i); + let ret = commonParts.join(path.sep); + if (!fs.existsSync(ret)) { + throw new Error(`Common directory does not exist: ${ret}`); + } + if (!fs.lstatSync(ret).isDirectory()) { + ret = path.dirname(ret); + } + return ret; +} + +function forwardCommand(args: string[]): number { + // Avoid infinite recursion + if (args.length == 0) { + console.error("No command provided"); + return 1; + } + const cmd = args[0]; + const envVariable = `__JUST_FORWARD_${cmd}`; + if (process.env[envVariable]) { + console.error(`No common ${cmd} handler found`); + return 1; + } + process.env[envVariable] = "true"; + const cmdArgs = args.slice(1); + const is_flag = /^(-.*|\++)$/; // + is used for testing level in some langauge tests + const flags = cmdArgs.filter(arg => is_flag.test(arg)); + const positionalArgs = cmdArgs.filter(arg => !is_flag.test(arg)); + + if (positionalArgs.length === 0) { + console.error("No positional arguments provided"); + return 1; + } + + const commonPath = commonDir(positionalArgs); + let relativeArgs = positionalArgs.map(arg => path.relative(commonPath, arg) || "."); + if (relativeArgs.length === 1 && relativeArgs[0] === ".") { + relativeArgs = []; + } + + const invocation = [process.env["JUST_EXECUTABLE"] || "just", cmd, ...flags, ...relativeArgs]; + console.log(`-> ${commonPath}: just ${invocation.slice(1).join(" ")}`); + try { + child_process.execFileSync(invocation[0], invocation.slice(1), { stdio: "inherit", cwd: commonPath }); + } catch (error) { + return 1; + } + return 0; +} + +process.exit(forwardCommand(process.argv.slice(2))); diff --git a/misc/just/forward.just b/misc/just/forward.just new file mode 100644 index 00000000000..fc3b1fde187 --- /dev/null +++ b/misc/just/forward.just @@ -0,0 +1,28 @@ +import "lib.just" + +# copy&paste necessary for each command until proper forwarding of multiple args is implemented +# see https://github.com/casey/just/issues/1988 + +_forward := tsx + source_dir() + "/forward-command.ts" + + +[no-cd, positional-arguments, no-exit-message] +@test *ARGS=".": + {{ _forward }} test "$@" + + +[no-cd, positional-arguments, no-exit-message] +@build *ARGS=".": + {{ _forward }} build "$@" + +[no-cd, positional-arguments, no-exit-message] +@generate *ARGS=".": + {{ _forward }} generate "$@" + +[no-cd, positional-arguments, no-exit-message] +@lint *ARGS=".": + {{ _forward }} lint "$@" + +[no-cd, positional-arguments, no-exit-message] +@format *ARGS=".": + {{ _forward }} format "$@" diff --git a/misc/just/lib.just b/misc/just/lib.just new file mode 100644 index 00000000000..8a6dc68f895 --- /dev/null +++ b/misc/just/lib.just @@ -0,0 +1,63 @@ +set fallback +set allow-duplicate-recipes +set allow-duplicate-variables +set unstable + +export PATH_SEP := if os() == "windows" { ";" } else { ":" } +export JUST_EXECUTABLE := just_executable() + +error := style("error") + "error" + NORMAL + ": " +cmd_sep := "\n#--------------------------------------------------------\n" +export CMD_BEGIN := style("command") + cmd_sep +export CMD_END := cmd_sep + NORMAL + +tsx := "npx tsx@4.19.0 " + +import? '../../../semmle-code.just' # internal repo just file, if present +import 'semmle-code-stub.just' + + +[no-exit-message] +@_require_semmle_code: + {{ if SEMMLE_CODE == "" { ''' + echo "''' + error + ''' running this recipe requires doing so from an internal repository checkout" >&2 + exit 1 + ''' } else { "" } }} + +_build LANGUAGE: _require_semmle_code (_maybe_build LANGUAGE) + +[no-exit-message] +_maybe_build LANGUAGE: + {{ cmd_sep }}{{ if SEMMLE_CODE == "" { '# using codeql from PATH, if any' } else { 'cd $SEMMLE_CODE; ./build target/intree/codeql-' + LANGUAGE } }}{{ cmd_sep }} + +[no-cd, positional-arguments, no-exit-message] +@_codeql_test LANGUAGE +ARGS: (_maybe_build LANGUAGE) + {{ tsx + source_dir() }}/codeql-test-run.ts "$@" + +[no-cd, positional-arguments, no-exit-message] +@_codeql_test_run_only LANGUAGE +ARGS: + {{ tsx + source_dir() }}/codeql-test-run.ts "$@" + + +[no-cd, no-exit-message] +_ql_format +ARGS: (_maybe_build "nolang") + {{ cmd_sep }}{{ if SEMMLE_CODE != "" { '$SEMMLE_CODE/target/intree/codeql-nolang/' } else { '' } }}codeql query format --in-place $(find {{ ARGS }} -type f -name '*.ql' -or -name '*.qll'){{ cmd_sep }} + + +[no-cd, no-exit-message] +_bazel COMMAND *ARGS: + {{ cmd_sep }}{{ if SEMMLE_CODE != "" { '$SEMMLE_CODE/tools/' } else { '' } }}bazel {{ COMMAND }} {{ ARGS }}{{ cmd_sep }} + +[no-cd, no-exit-message] +_integration_test *ARGS: _require_semmle_code (_run "$SEMMLE_CODE/tools/pytest" ARGS) + +[no-cd] +_run +ARGS: + {{ cmd_sep }}{{ ARGS }}{{ cmd_sep }} + +[no-cd, positional-arguments, no-exit-message] +_just +ARGS: + {{ JUST_EXECUTABLE }} "$@" + +[no-cd] +_black *ARGS=".": (_run "uv" "run" "black" ARGS) diff --git a/misc/just/semmle-code-stub.just b/misc/just/semmle-code-stub.just new file mode 100644 index 00000000000..14733ffb648 --- /dev/null +++ b/misc/just/semmle-code-stub.just @@ -0,0 +1 @@ +export SEMMLE_CODE := "" diff --git a/rust/justfile b/rust/justfile new file mode 100644 index 00000000000..bcd753087da --- /dev/null +++ b/rust/justfile @@ -0,0 +1,11 @@ +import '../misc/just/lib.just' + +install: (_bazel "run" "@codeql//rust:install") + +build: generate (_build "rust") + +generate: (_bazel "run" "@codeql//rust/codegen") + +lint: (_run "python3" "lint.py") + +format: (_run "python3" "lint.py" "--format-only") diff --git a/rust/lint.py b/rust/lint.py index 600a888649e..e3a078635fa 100755 --- a/rust/lint.py +++ b/rust/lint.py @@ -4,6 +4,14 @@ import subprocess import pathlib import shutil import sys +import argparse + + +def options(): + parser = argparse.ArgumentParser(description="lint rust language pack code") + parser.add_argument("--format-only", action="store_true", help="Only apply formatting") + return parser.parse_args() + def tool(name): @@ -12,27 +20,35 @@ def tool(name): return ret -this_dir = pathlib.Path(__file__).resolve().parent - -cargo = tool("cargo") -bazel = tool("bazel") - -runs = [] +def main(): + args = options() + this_dir = pathlib.Path(__file__).resolve().parent -def run(tool, args, *, cwd=this_dir): - print("+", tool, args) - runs.append(subprocess.run([tool] + args.split(), cwd=cwd)) + cargo = tool("cargo") + bazel = tool("bazel") + + runs = [] -# make sure bazel-provided sources are put in tree for `cargo` to work with them -run(bazel, "run ast-generator:inject-sources") -run(cargo, "fmt --all --quiet") + def run(tool, args, *, cwd=this_dir): + print("+", tool, args) + runs.append(subprocess.run([tool] + args.split(), cwd=cwd)) -for manifest in this_dir.rglob("Cargo.toml"): - if not manifest.is_relative_to(this_dir / "ql") and not manifest.is_relative_to(this_dir / "integration-tests"): - run(cargo, - "clippy --fix --allow-dirty --allow-staged --quiet -- -D warnings", - cwd=manifest.parent) -sys.exit(max(r.returncode for r in runs)) + # make sure bazel-provided sources are put in tree for `cargo` to work with them + run(bazel, "run ast-generator:inject-sources") + run(cargo, "fmt --all --quiet") + + if not args.format_only: + for manifest in this_dir.rglob("Cargo.toml"): + if not manifest.is_relative_to(this_dir / "ql") and not manifest.is_relative_to(this_dir / "integration-tests"): + run(cargo, + "clippy --fix --allow-dirty --allow-staged --quiet -- -D warnings", + cwd=manifest.parent) + + return max(r.returncode for r in runs) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/rust/ql/integration-tests/justfile b/rust/ql/integration-tests/justfile new file mode 100644 index 00000000000..c8fad074a3f --- /dev/null +++ b/rust/ql/integration-tests/justfile @@ -0,0 +1,9 @@ +import "../../../misc/just/lib.just" + + +[no-cd] +test *ARGS=".": (_just "generate") (_integration_test ARGS) + +# TODO in separate PR +# [no-cd] +# format *ARGS=".": (_ql_format ARGS) (_black ARGS) diff --git a/rust/ql/justfile b/rust/ql/justfile new file mode 100644 index 00000000000..d4f2aa8fc82 --- /dev/null +++ b/rust/ql/justfile @@ -0,0 +1,4 @@ +import "../../misc/just/lib.just" + +[no-cd] +format *ARGS=".": (_ql_format ARGS) diff --git a/rust/ql/test/justfile b/rust/ql/test/justfile new file mode 100644 index 00000000000..c550af68a14 --- /dev/null +++ b/rust/ql/test/justfile @@ -0,0 +1,17 @@ +import '../../../misc/just/lib.just' + +plus := """ + --check-databases + --check-diff-informed + --fail-on-trap-errors + --check-undefined-labels + --check-unused-labels + --check-repeated-labels + --check-redefined-labels + --check-use-before-definition +""" + +plusplus := "--consistency-queries=" + source_dir() + "/../consistency-queries" + +[no-cd] +test *ARGS=".": (_codeql_test "rust" ARGS plus plusplus) \ No newline at end of file