diff --git a/misc/just/codeql-test-run.ts b/misc/just/codeql-test-run.ts deleted file mode 100644 index 7c9d057425f..00000000000 --- a/misc/just/codeql-test-run.ts +++ /dev/null @@ -1,159 +0,0 @@ -import * as child_process from "child_process"; -import * as path from "path"; -import * as os from "os"; -import * as fs from "fs"; - -const vars = { - just: process.env["JUST_EXECUTABLE"] || "just", - error: process.env["JUST_ERROR"] || "error", - cmd_begin: process.env["CMD_BEGIN"] || "", - cmd_end: process.env["CMD_END"] || "", - semmle_code: process.env["SEMMLE_CODE"], -}; - -function invoke( - invocation: string[], - options: { cwd?: string; log_prefix?: string } = {}, -): number { - const log_prefix = - options.log_prefix && options.log_prefix !== "" - ? `${options.log_prefix} ` - : ""; - console.log( - `${vars.cmd_begin}${log_prefix}${invocation.join(" ")}${vars.cmd_end}`, - ); - try { - child_process.execFileSync(invocation[0], invocation.slice(1), { - stdio: "inherit", - cwd: options.cwd, - }); - } catch (error) { - return 1; - } - return 0; -} - -type Args = { - tests: string[]; - flags: string[]; - env: string[]; - codeql: string; - all: boolean; -}; - -const old_console_error = console.error; - -console.error = (message: string) => { - old_console_error(vars.error + message); -}; - -function parseArgs(args: Args, argv: string) { - argv.split(/(? arg.replace("\\ ", " ")) - .forEach((arg) => { - if (arg.startsWith("--codeql=")) { - args.codeql = arg.split("=")[1]; - } else if (arg === "+" || arg === "--all-checks") { - args.all = true; - } else if (arg.startsWith("-")) { - args.flags.push(arg); - } else if (/^[A-Z_][A-Z_0-9]*=.*$/.test(arg)) { - args.env.push(arg); - } else if (arg !== "") { - args.tests.push(arg); - } - }); -} - -function codeqlTestRun(argv: string[]): number { - const [language, base_args, all_args, extra_args] = argv; - const ram_per_thread = process.platform === "linux" ? 3000 : 2048; - const cpus = os.cpus().length; - let args: Args = { - tests: [], - flags: [`--ram=${ram_per_thread * cpus}`, `-j${cpus}`], - env: [], - codeql: vars.semmle_code ? "build" : "host", - all: false, - }; - parseArgs(args, base_args); - parseArgs(args, extra_args); - if (args.all) { - parseArgs(args, all_args); - } - if ( - !vars.semmle_code && - (args.codeql === "build" || args.codeql === "built") - ) { - console.error( - "Using `--codeql=build` or `--codeql=built` requires working with the internal repository", - ); - return 1; - } - if (args.tests.length === 0) { - args.tests.push("."); - } - if (args.codeql === "build") { - if ( - invoke([vars.just, language, "build"], { - cwd: vars.semmle_code, - }) !== 0 - ) { - return 1; - } - } - if (args.codeql !== "host") { - // disable the default implicit config file, but keep an explicit one - // this is the same behavior wrt to `--codeql` as the integration test runner - process.env["CODEQL_CONFIG_FILE"] ||= "."; - } - // Set and unset environment variables - args.env.forEach((envVar) => { - const [key, value] = envVar.split("=", 2); - if (key) { - if (value === undefined) { - delete process.env[key]; - } else { - process.env[key] = value; - } - } else { - console.error(`Invalid environment variable assignment: ${envVar}`); - process.exit(1); - } - }); - let codeql; - function check_codeql() { - if (!fs.existsSync(codeql)) { - console.error(`CodeQL executable not found: ${codeql}`); - process.exit(1); - } - } - if (args.codeql === "built" || args.codeql === "build") { - codeql = path.join( - vars.semmle_code!, - "target", - "intree", - `codeql-${language}`, - "codeql", - ); - check_codeql(); - } else if (args.codeql === "host") { - codeql = "codeql"; - } else { - codeql = args.codeql; - check_codeql(); - } - if (fs.lstatSync(codeql).isDirectory()) { - codeql = path.join(codeql, "codeql"); - if (process.platform === "win32") { - codeql += ".exe"; - } - check_codeql(); - } - - return invoke([codeql, "test", "run", ...args.flags, "--", ...args.tests], { - log_prefix: args.env.join(" "), - }); -} - -process.exit(codeqlTestRun(process.argv.slice(2))); diff --git a/misc/just/codeql_test_run.py b/misc/just/codeql_test_run.py new file mode 100644 index 00000000000..97785739444 --- /dev/null +++ b/misc/just/codeql_test_run.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +"""Run CodeQL tests with appropriate configuration. + +Called from just recipes as: + python3 codeql_test_run.py LANGUAGE BASE_FLAGS ALL_CHECKS_FLAGS EXTRA_ARGS +""" + +import os +import re +import shlex +import subprocess +import sys +from pathlib import Path + +JUST = os.environ.get("JUST_EXECUTABLE", "just") +ERROR = os.environ.get("JUST_ERROR", "error: ") +CMD_BEGIN = os.environ.get("CMD_BEGIN", "") +CMD_END = os.environ.get("CMD_END", "") +SEMMLE_CODE = os.environ.get("SEMMLE_CODE") + + +def invoke(invocation, *, cwd=None, log_prefix=""): + prefix = f"{log_prefix} " if log_prefix else "" + print(f"{CMD_BEGIN}{prefix}{' '.join(invocation)}{CMD_END}") + try: + subprocess.run(invocation, check=True, cwd=cwd) + except subprocess.CalledProcessError as e: + return e.returncode + return 0 + + +def error(message): + print(f"{ERROR}{message}", file=sys.stderr) + + +ENV_RE = re.compile(r"^[A-Z_][A-Z_0-9]*=.*$") + + +def parse_args(args, argv_str): + """Parse a space-separated argument string into categorized arguments.""" + for arg in shlex.split(argv_str): + if arg.startswith("--codeql="): + args["codeql"] = arg.split("=", 1)[1] + elif arg in ("+", "--all-checks"): + args["all"] = True + elif arg.startswith("-"): + args["flags"].append(arg) + elif ENV_RE.match(arg): + args["env"].append(arg) + elif arg: + args["tests"].append(arg) + + +def main(): + argv = sys.argv[1:] + if len(argv) < 4: + error( + "Usage: codeql_test_run.py LANGUAGE BASE_FLAGS ALL_CHECKS_FLAGS EXTRA_ARGS" + ) + return 1 + + language, base_args, all_args, extra_args = argv[0], argv[1], argv[2], argv[3] + ram_per_thread = 3000 if sys.platform == "linux" else 2048 + cpus = os.cpu_count() or 1 + + args = { + "tests": [], + "flags": [f"--ram={ram_per_thread * cpus}", f"-j{cpus}"], + "env": [], + "codeql": "build" if SEMMLE_CODE else "host", + "all": False, + } + parse_args(args, base_args) + parse_args(args, extra_args) + if args["all"]: + parse_args(args, all_args) + + if not SEMMLE_CODE and args["codeql"] in ("build", "built"): + error( + "Using `--codeql=build` or `--codeql=built` requires working " + "with the internal repository" + ) + return 1 + + if not args["tests"]: + args["tests"].append(".") + + if args["codeql"] == "build": + if invoke([JUST, language, "build"], cwd=SEMMLE_CODE) != 0: + return 1 + + if args["codeql"] != "host": + # Disable the default implicit config file, but keep an explicit one. + # Same behavior wrt --codeql as the integration test runner. + os.environ.setdefault("CODEQL_CONFIG_FILE", ".") + + for env_var in args["env"]: + key, _, value = env_var.partition("=") + if not key: + error(f"Invalid environment variable assignment: {env_var}") + return 1 + os.environ[key] = value + + # Resolve codeql executable + if args["codeql"] in ("built", "build"): + codeql = Path(SEMMLE_CODE, "target", "intree", f"codeql-{language}", "codeql") + if not codeql.exists(): + error(f"CodeQL executable not found: {codeql}") + return 1 + elif args["codeql"] == "host": + codeql = Path("codeql") + else: + codeql = Path(args["codeql"]) + if not codeql.exists(): + error(f"CodeQL executable not found: {codeql}") + return 1 + + if codeql.is_dir(): + codeql = codeql / "codeql" + if sys.platform == "win32": + codeql = codeql.with_suffix(".exe") + if not codeql.exists(): + error(f"CodeQL executable not found: {codeql}") + return 1 + + return invoke( + [str(codeql), "test", "run", *args["flags"], "--", *args["tests"]], + log_prefix=" ".join(args["env"]), + ) + + +if __name__ == "__main__": + try: + sys.exit(main()) + except KeyboardInterrupt: + sys.exit(128 + 2) diff --git a/misc/just/defs.just b/misc/just/defs.just index 6f46b84597a..71cba34e1a7 100644 --- a/misc/just/defs.just +++ b/misc/just/defs.just @@ -15,7 +15,7 @@ export CMD_BEGIN := style("command") + cmd_sep export CMD_END := cmd_sep + NORMAL export JUST_ERROR := error -tsx := "npx tsx@4.19.0" +py := "python3" default_db_checks := """\ --check-databases \ diff --git a/misc/just/forward-command.ts b/misc/just/forward-command.ts deleted file mode 100644 index 3e7354d4da0..00000000000 --- a/misc/just/forward-command.ts +++ /dev/null @@ -1,131 +0,0 @@ -import * as child_process from "child_process"; -import * as path from "path"; -import * as fs from "fs"; - -const vars = { - just: process.env["JUST_EXECUTABLE"] || "just", - error: process.env["JUST_ERROR"] || "", -}; - -console.debug = (...args: any[]) => {}; // comment out to debug script -const old_console_error = console.error; -console.error = (message: string) => { - old_console_error(vars.error + message); -}; - -function checkJustCommand( - justfile: string, - command: string, - postitionalArgs: string[], -): boolean { - if (!fs.existsSync(justfile)) { - return false; - } - let { cwd, args } = getJustContext(justfile, command, [], postitionalArgs); - console.debug( - `Checking: ${cwd ? `cd ${cwd}; ` : ""}just ${args.join(", ")}`, - ); - const res = child_process.spawnSync(vars.just, ["--dry-run", ...args], { - stdio: ["ignore", "ignore", "pipe"], - encoding: "utf8", - cwd, - }); - console.debug("result:", res); - // avoid having the forwarder find itself - return ( - res.status === 0 && - !res.stderr.includes(`forward-command.ts" ${command} "$@"`) - ); -} - -function findJustfile(command: string, arg: string): string | undefined { - for (let p = arg; ; p = path.dirname(p)) { - const candidate = path.join(p, "justfile"); - if (checkJustCommand(candidate, command, [arg])) { - return candidate; - } - if (p === "/" || p === ".") { - return undefined; - } - } -} - -function forwardCommand(args: string[]): number { - if (args.length == 0) { - console.error("No command provided"); - return 1; - } - return forward(args[0], args.slice(1)); -} - -function forward(cmd: string, args: string[]): number { - // non-positional arguments are flags, + (used by language tests) or environment variable settings - const is_non_positional = /^(-.*|\+|[A-Z_][A-Z_0-9]*=.*)$/; - const flags = args.filter((arg) => is_non_positional.test(arg)); - const positionalArgs = args.filter((arg) => !is_non_positional.test(arg)); - let justfiles: Map = new Map(); - for (const arg of positionalArgs.length > 0 ? positionalArgs : ["."]) { - const justfile = findJustfile(cmd, arg); - if (!justfile) { - console.error(`No justfile found for ${cmd} on ${arg}`); - return 1; - } - justfiles.set(justfile, [...(justfiles.get(justfile) || []), arg]); - } - const invocations = Array.from(justfiles.entries()).map( - ([justfile, positionalArgs]) => { - const { cwd, args } = getJustContext( - justfile, - cmd, - flags, - positionalArgs, - ); - console.log(`-> ${cwd ? `cd ${cwd}; ` : ""}just ${args.join(" ")}`); - return { cwd, args }; - }, - ); - for (const { cwd, args } of invocations) { - if (invokeJust(cwd, args) !== 0) { - return 1; - } - } - return 0; -} - -function getJustContext( - justfile: string, - cmd: string, - flags: string[], - positionalArgs: string[], -): { args: string[]; cwd?: string } { - if ( - positionalArgs.length === 1 && - justfile == path.join(positionalArgs[0], "justfile") - ) { - // If there's only one positional argument and it matches the justfile path, suppress arguments - // so for example `just build ql/rust` becomes `just build` in the `ql/rust` directory - return { - cwd: positionalArgs[0], - args: [cmd, ...flags], - }; - } else { - return { - cwd: undefined, - args: ["--justfile", justfile, cmd, ...flags, ...positionalArgs], - }; - } -} - -function invokeJust(cwd: string | undefined, args: string[]): number { - try { - child_process.execFileSync(vars.just, args, { - stdio: "inherit", - cwd, - }); - } catch (error) { - return 1; - } - return 0; -} - -process.exit(forwardCommand(process.argv.slice(2))); diff --git a/misc/just/forward.just b/misc/just/forward.just index c61f043c4a8..8dd83e28496 100644 --- a/misc/just/forward.just +++ b/misc/just/forward.just @@ -7,7 +7,7 @@ import "lib.just" # copy&paste necessary for each command until proper forwarding of multiple args is implemented # see https://github.com/casey/just/issues/1988 -_forward := tsx + ' "' + source_dir() + '/forward-command.ts"' +_forward := py + ' "' + source_dir() + '/forward_command.py"' alias t := test alias b := build diff --git a/misc/just/forward_command.py b/misc/just/forward_command.py new file mode 100644 index 00000000000..d905ad5dde9 --- /dev/null +++ b/misc/just/forward_command.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +"""Forward commands to language-specific justfiles. + +Called from just recipes as: + python3 forward_command.py COMMAND [ARGS...] +""" + +import os +import re +import subprocess +import sys +from pathlib import Path + +JUST = os.environ.get("JUST_EXECUTABLE", "just") +ERROR = os.environ.get("JUST_ERROR", "") + + +def error(message): + print(f"{ERROR}{message}", file=sys.stderr) + + +def get_just_context(justfile, cmd, flags, positional_args): + """Get the (cwd, args) for invoking just with the given justfile.""" + if ( + len(positional_args) == 1 + and justfile == Path(positional_args[0]) / "justfile" + ): + # If there's only one positional argument and it matches the justfile + # path, suppress arguments so e.g. `just build ql/rust` becomes + # `just build` in the `ql/rust` directory + return positional_args[0], [cmd, *flags] + else: + return None, ["--justfile", str(justfile), cmd, *flags, *positional_args] + + +def check_just_command(justfile, command, positional_args): + """Check if a justfile supports the given command.""" + if not justfile.exists(): + return False + cwd, args = get_just_context(justfile, command, [], positional_args) + result = subprocess.run( + [JUST, "--dry-run", *args], + cwd=cwd, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + text=True, + ) + # Avoid having the forwarder find itself + return ( + result.returncode == 0 + and f'forward_command.py" {command} "$@"' not in result.stderr + ) + + +def find_justfile(command, arg): + """Search up the directory tree for a justfile supporting the command.""" + for p in [Path(arg), *Path(arg).parents]: + candidate = p / "justfile" + if check_just_command(candidate, command, [arg]): + return candidate + return None + + +def invoke_just(cwd, args): + """Run just with the given arguments.""" + try: + subprocess.run([JUST, *args], check=True, cwd=cwd) + except subprocess.CalledProcessError as e: + return e.returncode + return 0 + + +def forward(cmd, args): + """Forward a command to language-specific justfiles.""" + is_non_positional = re.compile(r"^(-.*|\+|[A-Z_][A-Z_0-9]*=.*)$") + flags = [arg for arg in args if is_non_positional.match(arg)] + positional_args = [arg for arg in args if not is_non_positional.match(arg)] + + justfiles = {} + for arg in positional_args or ["."]: + justfile = find_justfile(cmd, arg) + if not justfile: + error(f"No justfile found for {cmd} on {arg}") + return 1 + justfiles.setdefault(justfile, []).append(arg) + + invocations = [] + for justfile, pos_args in justfiles.items(): + cwd, just_args = get_just_context(justfile, cmd, flags, pos_args) + prefix = f"cd {cwd}; " if cwd else "" + print(f"-> {prefix}just {' '.join(just_args)}") + invocations.append((cwd, just_args)) + + for cwd, just_args in invocations: + if invoke_just(cwd, just_args) != 0: + return 1 + return 0 + + +def main(): + argv = sys.argv[1:] + if not argv: + error("No command provided") + return 1 + return forward(argv[0], argv[1:]) + + +if __name__ == "__main__": + try: + sys.exit(main()) + except KeyboardInterrupt: + sys.exit(128 + 2) diff --git a/misc/just/language-tests.ts b/misc/just/language-tests.ts deleted file mode 100644 index 31cb9ff564b..00000000000 --- a/misc/just/language-tests.ts +++ /dev/null @@ -1,33 +0,0 @@ -import * as path from "path"; -import * as process from "process"; -import * as child_process from "child_process"; - -function languageTests(argv: string[]): number { - const [extra_args, dir, ...relativeRoots] = argv; - const semmle_code = process.env["SEMMLE_CODE"]!; - let roots = relativeRoots.map((root) => - path.relative(semmle_code, path.join(dir, root)), - ); - const invocation = [ - process.env["JUST_EXECUTABLE"] || "just", - "--justfile", - path.join(roots[0], "justfile"), - "test", - "--all-checks", - "--codeql=built", - ...extra_args.split(" "), - ...roots, - ]; - console.log(`-> just ${invocation.slice(1).join(" ")}`); - try { - child_process.execFileSync(invocation[0], invocation.slice(1), { - stdio: "inherit", - cwd: semmle_code, - }); - } catch (error) { - return 1; - } - return 0; -} - -process.exit(languageTests(process.argv.slice(2))); diff --git a/misc/just/language_tests.py b/misc/just/language_tests.py new file mode 100644 index 00000000000..4b9d12ab45e --- /dev/null +++ b/misc/just/language_tests.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +"""Run all language tests for CI. + +Called from just recipes as: + python3 language_tests.py EXTRA_ARGS SOURCE_DIR ROOT1 [ROOT2 ...] +""" + +import os +import subprocess +import sys +from pathlib import Path + + +def main(): + argv = sys.argv[1:] + if len(argv) < 3: + print( + "Usage: language_tests.py EXTRA_ARGS SOURCE_DIR ROOT1 [ROOT2 ...]", + file=sys.stderr, + ) + return 1 + + extra_args, source_dir, *relative_roots = argv + + semmle_code = Path(os.environ["SEMMLE_CODE"]) + roots = [ + os.path.relpath(Path(source_dir) / root, semmle_code) + for root in relative_roots + ] + + just = os.environ.get("JUST_EXECUTABLE", "just") + invocation = [ + just, + "--justfile", + str(Path(roots[0]) / "justfile"), + "test", + "--all-checks", + "--codeql=built", + *(a for a in extra_args.split(" ") if a), + *roots, + ] + + print(f"-> just {' '.join(invocation[1:])}") + try: + subprocess.run(invocation, check=True, cwd=semmle_code) + except subprocess.CalledProcessError as e: + return e.returncode + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except KeyboardInterrupt: + sys.exit(128 + 2) diff --git a/misc/just/lib.just b/misc/just/lib.just index 2c57281e89a..d3fbe25f551 100644 --- a/misc/just/lib.just +++ b/misc/just/lib.just @@ -7,12 +7,12 @@ import "format.just" # `--all-checks` or `+` is passed along in `EXTRA_ARGS` [no-cd, positional-arguments, no-exit-message] @_codeql_test LANGUAGE BASE_FLAGS ALL_CHECKS_FLAGS EXTRA_ARGS: - {{ tsx }} "{{ source_dir() }}/codeql-test-run.ts" "$@" + {{ py }} "{{ source_dir() }}/codeql_test_run.py" "$@" # Run all language tests in ROOTS for a language. This is intended to be called by CI [no-cd, positional-arguments, no-exit-message] @_language_tests EXTRA_ARGS SOURCE_DIR +ROOTS: _require_semmle_code - {{ tsx }} "{{ source_dir() }}/language-tests.ts" "$@" + {{ py }} "{{ source_dir() }}/language_tests.py" "$@" # Run integration tests. Requires an internal repository checkout [no-cd, no-exit-message]