Just: introduce scaffolding for common verbs, and apply to rust

justfile

@@ -0,0 +1,2 @@
+import 'misc/just/lib.just'
+import 'misc/just/forward.just'

misc/bazel/justfile

@@ -0,0 +1,5 @@
+import '../just/lib.just'
+
+[no-cd, positional-arguments, no-exit-message]
+hello +ARGS:
+    @echo "hello from bzl" "$@"

misc/just/codeql-test-run.ts

@@ -0,0 +1,39 @@
+import * as child_process from "child_process";
+import * as path from "path";
+
+function codeqlTestRun(argv: string[]): number {
+    const [language, args, ...plus] = argv;
+    let codeql =
+    process.env["SEMMLE_CODE"] ?
+        path.join(process.env["SEMMLE_CODE"], "target", "intree", `codeql-${language}`, "codeql")
+    :
+        "codeql"
+    ;
+    process.env["CODEQL_CONFIG_FILE"] ||= "."  // disable the default implicit config file, but keep an explicit one
+    let plus_options = plus.map(option => option.trim().split("\n").filter(option => option !== ""));
+    let testing_level = 0;
+    let parsed_args = args.split(" ").filter(arg => {
+        if (arg === "") return false;
+        if (/^\++$/.test(arg)) {
+            testing_level = Math.max(testing_level, arg.length);
+            return false;
+        }
+        return true;
+    });
+    if (parsed_args.every(arg => arg.startsWith("-"))) {
+        parsed_args.push(".");
+    }
+    let invocation = [codeql, "test", "run", "-j0", ...parsed_args];
+    for (let i = 0; i < Math.min(plus_options.length, testing_level); i++) {
+        invocation.push(...plus_options[i]);
+    }
+    console.log(`${process.env["CMD_BEGIN"] || ""}${invocation.join(" ")}${process.env["CMD_END"] || ""}`);
+    try {
+        child_process.execFileSync(invocation[0], invocation.slice(1), { stdio: "inherit" });
+    } catch (error) {
+        return 1;
+    }
+    return 0;
+}
+
+process.exit(codeqlTestRun(process.argv.slice(2)));

misc/just/forward-command.ts

@@ -0,0 +1,64 @@
+import * as child_process from "child_process";
+import * as path from "path";
+import * as fs from "fs";
+
+function commonDir(paths: string[]): string {
+  if (paths.length === 0) return "";
+  const splitPaths = paths.map(p => p.split(path.sep));
+  let i;
+  for (i = 0; i < splitPaths[0].length; i++) {
+    if (!splitPaths.every(parts => parts[i] === splitPaths[0][i])) {
+      break;
+    }
+    }
+    const commonParts = splitPaths[0].slice(0, i);
+  let ret = commonParts.join(path.sep);
+  if (!fs.existsSync(ret)) {
+    throw new Error(`Common directory does not exist: ${ret}`);
+  }
+  if (!fs.lstatSync(ret).isDirectory()) {
+    ret = path.dirname(ret);
+  }
+  return ret;
+}
+
+function forwardCommand(args: string[]): number {
+  // Avoid infinite recursion
+  if (args.length == 0) {
+    console.error("No command provided");
+    return 1;
+  }
+  const cmd = args[0];
+  const envVariable = `__JUST_FORWARD_${cmd}`;
+  if (process.env[envVariable]) {
+    console.error(`No common ${cmd} handler found`);
+    return 1;
+  }
+  process.env[envVariable] = "true";
+  const cmdArgs = args.slice(1);
+  const is_flag = /^(-.*|\++)$/;  // + is used for testing level in some langauge tests
+  const flags = cmdArgs.filter(arg => is_flag.test(arg));
+  const positionalArgs = cmdArgs.filter(arg => !is_flag.test(arg));
+
+  if (positionalArgs.length === 0) {
+    console.error("No positional arguments provided");
+    return 1;
+  }
+
+  const commonPath = commonDir(positionalArgs);
+  let relativeArgs = positionalArgs.map(arg => path.relative(commonPath, arg) || ".");
+  if (relativeArgs.length === 1 && relativeArgs[0] === ".") {
+    relativeArgs = [];
+  }
+
+  const invocation = [process.env["JUST_EXECUTABLE"] || "just", cmd, ...flags, ...relativeArgs];
+  console.log(`-> ${commonPath}: just ${invocation.slice(1).join(" ")}`);
+  try {
+    child_process.execFileSync(invocation[0], invocation.slice(1), { stdio: "inherit", cwd: commonPath });
+  } catch (error) {
+    return 1;
+  }
+  return 0;
+}
+
+process.exit(forwardCommand(process.argv.slice(2)));

misc/just/forward.just

@@ -0,0 +1,28 @@
+import "lib.just"
+
+# copy&paste necessary for each command until proper forwarding of multiple args is implemented
+# see https://github.com/casey/just/issues/1988
+
+_forward := tsx + source_dir() + "/forward-command.ts"
+
+
+[no-cd, positional-arguments, no-exit-message]
+@test *ARGS=".":
+    {{ _forward }} test "$@"
+
+
+[no-cd, positional-arguments, no-exit-message]
+@build *ARGS=".":
+    {{ _forward }} build "$@"
+
+[no-cd, positional-arguments, no-exit-message]
+@generate *ARGS=".":
+    {{ _forward }} generate "$@"
+
+[no-cd, positional-arguments, no-exit-message]
+@lint *ARGS=".":
+    {{ _forward }} lint "$@"
+
+[no-cd, positional-arguments, no-exit-message]
+@format *ARGS=".":
+    {{ _forward }} format "$@"

misc/just/lib.just

@@ -0,0 +1,63 @@
+set fallback
+set allow-duplicate-recipes
+set allow-duplicate-variables
+set unstable
+
+export PATH_SEP := if os() == "windows" { ";" } else { ":" }
+export JUST_EXECUTABLE := just_executable()
+
+error := style("error") + "error" + NORMAL + ": "
+cmd_sep := "\n#--------------------------------------------------------\n"
+export CMD_BEGIN := style("command") + cmd_sep
+export CMD_END := cmd_sep + NORMAL
+
+tsx := "npx tsx@4.19.0 "
+
+import? '../../../semmle-code.just'  # internal repo just file, if present
+import 'semmle-code-stub.just'
+
+
+[no-exit-message]
+@_require_semmle_code:
+    {{ if SEMMLE_CODE == "" { '''
+        echo "''' + error + ''' running this recipe requires doing so from an internal repository checkout" >&2
+        exit 1
+    ''' } else { "" } }}
+
+_build LANGUAGE: _require_semmle_code (_maybe_build LANGUAGE)
+
+[no-exit-message]
+_maybe_build LANGUAGE:
+    {{ cmd_sep }}{{ if SEMMLE_CODE == "" { '# using codeql from PATH, if any' } else { 'cd $SEMMLE_CODE; ./build target/intree/codeql-' + LANGUAGE } }}{{ cmd_sep }}
+
+[no-cd, positional-arguments, no-exit-message]
+@_codeql_test LANGUAGE +ARGS: (_maybe_build LANGUAGE)
+    {{ tsx + source_dir() }}/codeql-test-run.ts "$@"
+
+[no-cd, positional-arguments, no-exit-message]
+@_codeql_test_run_only LANGUAGE +ARGS:
+    {{ tsx + source_dir() }}/codeql-test-run.ts "$@"
+
+
+[no-cd, no-exit-message]
+_ql_format +ARGS: (_maybe_build "nolang")
+    {{ cmd_sep }}{{ if SEMMLE_CODE != "" { '$SEMMLE_CODE/target/intree/codeql-nolang/' } else { '' } }}codeql query format --in-place $(find {{ ARGS }} -type f -name '*.ql' -or -name '*.qll'){{ cmd_sep }}
+
+
+[no-cd, no-exit-message]
+_bazel COMMAND *ARGS:
+    {{ cmd_sep }}{{ if SEMMLE_CODE != "" { '$SEMMLE_CODE/tools/' } else { '' } }}bazel {{ COMMAND }} {{ ARGS }}{{ cmd_sep }}
+
+[no-cd, no-exit-message]
+_integration_test *ARGS: _require_semmle_code (_run "$SEMMLE_CODE/tools/pytest" ARGS)
+
+[no-cd]
+_run +ARGS:
+    {{ cmd_sep }}{{ ARGS }}{{ cmd_sep }}
+
+[no-cd, positional-arguments, no-exit-message]
+_just +ARGS:
+    {{ JUST_EXECUTABLE }} "$@"
+
+[no-cd]
+_black *ARGS=".": (_run "uv" "run" "black" ARGS)

misc/just/semmle-code-stub.just

@@ -0,0 +1 @@
+export SEMMLE_CODE := ""

rust/justfile

@@ -0,0 +1,11 @@
+import '../misc/just/lib.just'
+
+install: (_bazel "run" "@codeql//rust:install")
+
+build: generate (_build "rust")
+
+generate: (_bazel "run" "@codeql//rust/codegen")
+
+lint: (_run "python3" "lint.py")
+
+format: (_run "python3" "lint.py" "--format-only")

rust/lint.py

@@ -4,6 +4,14 @@ import subprocess
 import pathlib
 import shutil
 import sys
+import argparse
+
+
+def options():
+    parser = argparse.ArgumentParser(description="lint rust language pack code")
+    parser.add_argument("--format-only", action="store_true", help="Only apply formatting")
+    return parser.parse_args()
+
 
 
 def tool(name):
@@ -12,27 +20,35 @@ def tool(name):
     return ret
 
 
-this_dir = pathlib.Path(__file__).resolve().parent
+def main():
-
+    args = options()
-cargo = tool("cargo")
+    this_dir = pathlib.Path(__file__).resolve().parent
-bazel = tool("bazel")
-
-runs = []
 
 
-def run(tool, args, *, cwd=this_dir):
+    cargo = tool("cargo")
-    print("+", tool, args)
+    bazel = tool("bazel")
-    runs.append(subprocess.run([tool] + args.split(), cwd=cwd))
+
+    runs = []
 
 
-# make sure bazel-provided sources are put in tree for `cargo` to work with them
+    def run(tool, args, *, cwd=this_dir):
-run(bazel, "run ast-generator:inject-sources")
+        print("+", tool, args)
-run(cargo, "fmt --all --quiet")
+        runs.append(subprocess.run([tool] + args.split(), cwd=cwd))
 
-for manifest in this_dir.rglob("Cargo.toml"):
-    if not manifest.is_relative_to(this_dir / "ql") and not manifest.is_relative_to(this_dir / "integration-tests"):
-        run(cargo,
-            "clippy --fix --allow-dirty --allow-staged --quiet -- -D warnings",
-            cwd=manifest.parent)
 
-sys.exit(max(r.returncode for r in runs))
+    # make sure bazel-provided sources are put in tree for `cargo` to work with them
+    run(bazel, "run ast-generator:inject-sources")
+    run(cargo, "fmt --all --quiet")
+
+    if not args.format_only:
+        for manifest in this_dir.rglob("Cargo.toml"):
+            if not manifest.is_relative_to(this_dir / "ql") and not manifest.is_relative_to(this_dir / "integration-tests"):
+                run(cargo,
+                    "clippy --fix --allow-dirty --allow-staged --quiet -- -D warnings",
+                    cwd=manifest.parent)
+
+    return max(r.returncode for r in runs)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

rust/ql/integration-tests/justfile

@@ -0,0 +1,9 @@
+import "../../../misc/just/lib.just"
+
+
+[no-cd]
+test *ARGS=".": (_just "generate") (_integration_test ARGS)
+
+# TODO in separate PR
+# [no-cd]
+# format *ARGS=".": (_ql_format ARGS) (_black ARGS)

rust/ql/justfile

@@ -0,0 +1,4 @@
+import "../../misc/just/lib.just"
+
+[no-cd]
+format *ARGS=".": (_ql_format ARGS)

rust/ql/test/justfile

@@ -0,0 +1,17 @@
+import '../../../misc/just/lib.just'
+
+plus := """
+    --check-databases
+    --check-diff-informed
+    --fail-on-trap-errors
+    --check-undefined-labels
+    --check-unused-labels
+    --check-repeated-labels
+    --check-redefined-labels
+    --check-use-before-definition
+"""
+
+plusplus := "--consistency-queries=" + source_dir() + "/../consistency-queries"
+
+[no-cd]
+test *ARGS=".": (_codeql_test "rust" ARGS plus plusplus)