Merge remote-tracking branch 'origin/main' into redsun82/just2

The parallel attribute causes concurrent sembuild calls that conflict
with each other.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

actions/justfile

@@ -0,0 +1,7 @@
+import '../lib.just'
+
+[group('build')]
+build: (_build_dist "actions")
+
+[group('test')]
+language-tests *EXTRA_ARGS: (_language_tests EXTRA_ARGS source_dir() 'ql/test')

actions/ql/integration-tests/justfile

@@ -0,0 +1,4 @@
+import "../../../lib.just"
+
+[no-cd]
+test *ARGS=".": (_integration_test ARGS)

actions/ql/justfile

@@ -0,0 +1,6 @@
+import "../../lib.just"
+
+[no-cd]
+format *ARGS=".": (_format_ql ARGS)
+
+consistency_queries := ""

actions/ql/test/justfile

@@ -0,0 +1,8 @@
+import "../justfile"
+
+base_flags := ""
+
+all_checks := default_db_checks
+
+[no-cd]
+test *ARGS=".": (_codeql_test "actions" base_flags all_checks ARGS)

cpp/justfile

@@ -0,0 +1,8 @@
+import '../lib.just'
+import? '../../cpp-coding-standards.just'
+
+[group('build')]
+build: (_build_dist "cpp")
+
+[group('test')]
+language-tests *EXTRA_ARGS: (_language_tests EXTRA_ARGS source_dir() 'ql/test' '../../semmlecode-cpp-tests')

cpp/ql/consistency-queries/badLocations.ql

@@ -0,0 +1,9 @@
+import cpp
+
+// Locations should either be :0:0:0:0 locations (UnknownLocation, or
+// a whole file), or all 4 fields should be positive.
+from Location l
+where
+  [l.getStartLine(), l.getEndLine(), l.getStartColumn(), l.getEndColumn()] != 0 and
+  [l.getStartLine(), l.getEndLine(), l.getStartColumn(), l.getEndColumn()] < 1
+select l

cpp/ql/consistency-queries/nullInToString.ql

@@ -0,0 +1,5 @@
+import cpp
+
+from Element e
+where e.toString().matches("%(null)%")
+select e

cpp/ql/consistency-queries/qlpack.yml

@@ -0,0 +1,5 @@
+name: codeql/cpp-consistency-queries
+groups: [cpp, test, consistency-queries]
+dependencies:
+  codeql/cpp-all: ${workspace}
+extractor: cpp

cpp/ql/consistency-queries/unusedLocations.ql

@@ -0,0 +1,10 @@
+import cpp
+
+from Location l
+where
+  not any(Element e).getLocation() = l and
+  not any(LambdaCapture lc).getLocation() = l and
+  not any(MacroAccess ma).getActualLocation() = l and
+  not any(NamespaceDeclarationEntry nde).getBodyLocation() = l and
+  not any(XmlLocatable xml).getLocation() = l
+select l

cpp/ql/consistency-queries/variableDeclarationsWithoutTypes.ql

@@ -0,0 +1,5 @@
+import cpp
+
+from VariableDeclarationEntry i
+where not exists(i.getType())
+select i

cpp/ql/consistency-queries/variablesWithoutTypes.ql

@@ -0,0 +1,5 @@
+import cpp
+
+from Variable i
+where not exists(i.getType())
+select i

cpp/ql/integration-tests/justfile

@@ -0,0 +1,4 @@
+import "../../../lib.just"
+
+[no-cd]
+test *ARGS=".": (_integration_test ARGS)

cpp/ql/justfile

@@ -0,0 +1,6 @@
+import "../../lib.just"
+
+[no-cd]
+format *ARGS=".": (_format_ql ARGS)
+
+consistency_queries := source_dir() / "consistency-queries"

cpp/ql/test/justfile

@@ -0,0 +1,12 @@
+import "../justfile"
+
+base_flags := "--include-location-in-star"
+
+all_checks := f"""\
+    {{default_db_checks}}\
+    --check-undefined-labels \
+    --check-unused-labels \
+    --consistency-queries={{consistency_queries}}"""
+
+[no-cd]
+test *ARGS=".": (_codeql_test "cpp" base_flags all_checks ARGS)

csharp/justfile

@@ -0,0 +1,7 @@
+import '../lib.just'
+
+[group('build')]
+build: (_build_dist "csharp")
+
+[group('test')]
+language-tests *EXTRA_ARGS: (_language_tests EXTRA_ARGS source_dir() 'ql/test')

csharp/ql/integration-tests/justfile

@@ -0,0 +1,4 @@
+import "../../../lib.just"
+
+[no-cd]
+test *ARGS=".": (_integration_test ARGS)

csharp/ql/justfile

@@ -0,0 +1,6 @@
+import "../../lib.just"
+
+[no-cd]
+format *ARGS=".": (_format_ql ARGS)
+
+consistency_queries := source_dir() / "consistency-queries"

csharp/ql/test/justfile

@@ -0,0 +1,14 @@
+import "../justfile"
+
+base_flags := ""
+
+all_checks := f"""\
+    {{default_db_checks}}\
+    --check-undefined-labels \
+    --check-repeated-labels \
+    --check-redefined-labels \
+    --additional-packs=ql \
+    --consistency-queries={{consistency_queries}}"""
+
+[no-cd]
+test *ARGS=".": (_codeql_test "csharp" base_flags all_checks ARGS)

go/justfile

@@ -0,0 +1,7 @@
+import '../lib.just'
+
+[group('build')]
+build: (_build_dist "go")
+
+[group('test')]
+language-tests *EXTRA_ARGS: (_language_tests EXTRA_ARGS source_dir() 'ql/test')

go/ql/integration-tests/justfile

@@ -0,0 +1,4 @@
+import "../../../lib.just"
+
+[no-cd]
+test *ARGS=".": (_integration_test ARGS)

go/ql/justfile

@@ -0,0 +1,6 @@
+import "../../lib.just"
+
+[no-cd]
+format *ARGS=".": (_format_ql ARGS)
+
+consistency_queries := source_dir() / "consistency-queries"

go/ql/test/justfile

@@ -0,0 +1,15 @@
+import "../justfile"
+
+base_flags := ""
+
+all_checks := f"""\
+    {{default_db_checks}}\
+    --check-undefined-labels \
+    --check-unused-labels \
+    --check-repeated-labels \
+    --check-redefined-labels \
+    --check-use-before-definition \
+    --consistency-queries={{consistency_queries}}"""
+
+[no-cd]
+test *ARGS=".": (_codeql_test "go" base_flags all_checks ARGS)

java/justfile

@@ -0,0 +1,3 @@
+import '../lib.just'
+
+build: (_build_dist "java")

java/ql/integration-tests/justfile

@@ -0,0 +1,4 @@
+import "../../../lib.just"
+
+[no-cd]
+test *ARGS=".": (_integration_test ARGS)

java/ql/justfile

@@ -0,0 +1,6 @@
+import "../../lib.just"
+
+[no-cd]
+format *ARGS=".": (_format_ql ARGS)
+
+consistency_queries := source_dir() / "consistency-queries"

java/ql/test-kotlin1/justfile

@@ -0,0 +1,16 @@
+import "../justfile"
+
+base_flags := """\
+    CODEQL_EXTRACTOR_KOTLIN_DIAGNOSTIC_LIMIT= \
+"""
+
+all_checks := f"""\
+    {{default_db_checks}}\
+    --check-undefined-labels \
+    --check-repeated-labels \
+    --check-redefined-labels \
+    --check-use-before-definition \
+    --consistency-queries={{consistency_queries}}"""
+
+[no-cd]
+test *ARGS=".": (_codeql_test "java" base_flags all_checks ARGS)

java/ql/test-kotlin2/justfile

@@ -0,0 +1,17 @@
+import "../justfile"
+
+base_flags := """\
+    CODEQL_EXTRACTOR_KOTLIN_DIAGNOSTIC_LIMIT= \
+    CODEQL_KOTLIN_LEGACY_TEST_EXTRACTION_KOTLIN2=true \
+"""
+
+all_checks := f"""\
+    {{default_db_checks}}\
+    --check-undefined-labels \
+    --check-repeated-labels \
+    --check-redefined-labels \
+    --check-use-before-definition \
+    --consistency-queries={{consistency_queries}}"""
+
+[no-cd]
+test *ARGS=".": (_codeql_test "java" base_flags all_checks ARGS)

java/ql/test/justfile

@@ -0,0 +1,17 @@
+import "../justfile"
+
+base_flags := """\
+     CODEQL_EXTRACTOR_KOTLIN_DIAGNOSTIC_LIMIT="\\ " \
+"""
+
+all_checks := f"""\
+    {{default_db_checks}}\
+    --check-undefined-labels \
+    --check-repeated-labels \
+    --check-redefined-labels \
+    --check-use-before-definition \
+    --consistency-queries={{consistency_queries}}"""
+
+[no-cd]
+test *ARGS=".": (_codeql_test "java" base_flags all_checks ARGS)
+

javascript/justfile

@@ -0,0 +1,7 @@
+import '../lib.just'
+
+[group('build')]
+build: (_build_dist "javascript")
+
+[group('test')]
+language-tests *EXTRA_ARGS: (_language_tests EXTRA_ARGS source_dir() 'ql/test')

javascript/ql/integration-tests/justfile

@@ -0,0 +1,4 @@
+import "../../../lib.just"
+
+[no-cd]
+test *ARGS=".": (_integration_test ARGS)

javascript/ql/justfile

@@ -0,0 +1,6 @@
+import "../../lib.just"
+
+[no-cd]
+format *ARGS=".": (_format_ql ARGS)
+
+consistency_queries := ""

javascript/ql/test/justfile

@@ -0,0 +1,8 @@
+import "../justfile"
+
+base_flags := ""
+
+all_checks := default_db_checks
+
+[no-cd]
+test *ARGS=".": (_codeql_test "javascript" base_flags all_checks ARGS)

justfile

@@ -0,0 +1,4 @@
+# see misc/just/README.md for an overview
+
+import 'lib.just'
+import 'misc/just/forward.just'

lib.just

@@ -0,0 +1 @@
+import "misc/just/lib.just"

misc/bazel/justfile

@@ -0,0 +1,5 @@
+import '../just/lib.just'
+
+[no-cd, positional-arguments, no-exit-message]
+hello +ARGS:
+    @echo "hello from bzl" "$@"

misc/codegen/justfile

@@ -0,0 +1,5 @@
+import "../just/lib.just"
+
+test *ARGS="": (_bazel "test" "@codeql//misc/codegen/...")
+
+format *ARGS=".": (_format_py ARGS)

misc/just/README.md

@@ -0,0 +1,44 @@
+This directory contains an infrastructure for [`just`](https://github.com/casey/just)
+recipes that can be used throughout this and the internal repository. In particular we
+have common verbs (`build`, `test`, `format`, `lint`, `generate`) that individual parts
+of the project can implement, and some common functionality that can be used to that
+effect.
+
+# Forwarding
+
+The core of the functionality is given by forwarding. The idea is that:
+
+- if you are in the directory where a verb is implemented, you will get that as per
+  standard `just` behaviour (possibly using fallback).
+- if on the other hand you are beneath it, and you run something like
+  `just test ql/rust/ql/test/{a,b}`, then a forwarder script finds a common justfile
+  implementing the verb for all the positional arguments passed there, and then retries
+  calling `just test` from there. So if `test` is implemented beneath that (in that case,
+  it is in `rust/ql/test`), it uses that recipe.
+- even if there isn't a recipe that is common to all the positional arguments, the
+  forwarder will still group the arguments in batches using the same recipe. So
+  `just build ql/rust ql/java`, or
+  `just test ql/rust/ql/test/some/language/test ql/rust/ql/integration-test/some/integration/test`
+  will also work, with corresponding recipes run sequentially.
+
+Another point is how launching QL tests can be tweaked:
+
+- by default, the corresponding CLI is built from the internal repo (nothing is done if
+  working in `codeql` standalone), and no additional database or consistency checks are
+  made
+- `--codeql=built` can be passed to skip the build step (if no changes were made to the
+  CLI/extractors). This is consistent with the same pytest option
+- you can add the additional checks that CI does with `--all-checks` or the `+`
+  abbreviation. These additional checks are configured in justfiles per language, and
+  correspond to all the additional checks that CI adds (but that a dev might not want to
+  run by default).
+
+Some caveats:
+
+- passing arguments with spaces generally doesn't work, although setting arguments with
+  spaces in `justfile`s (for the base arguments) is supported using escaping as in `\\`.
+  This is a known limitation of just (see
+  <https://github.com/casey/just/issues/1988>)
+- when running different recipes for the same verb, non-positional arguments need to be
+  supported by all recipes involved. For example, this will work ok for `--learn` or
+  `--codeql` options in language and integration tests

misc/just/build.just

@@ -0,0 +1,19 @@
+# Helper build recipes
+
+import "defs.just"
+
+# Build the given language-specific CLI distribution
+_build_dist LANGUAGE: _require_semmle_code (_maybe_build_dist LANGUAGE)
+
+# Build the language-specific distribution if we are in an internal repository checkout
+# Otherwise, do nothing
+[no-exit-message]
+_maybe_build_dist LANGUAGE: (_if_in_semmle_code (f'cd "$SEMMLE_CODE"; MSYS2_ARG_CONV_EXCL="*" tools/bazel test //language-packs:intree-{{LANGUAGE}}-as-test --test_output=all') '# using codeql from PATH, if any')
+
+# Call bazel. Uses our official bazel wrapper if we are in an internal repository checkout
+[no-cd, no-exit-message]
+_bazel COMMAND *ARGS: (_if_in_semmle_code 'cd "$SEMMLE_CODE"; MSYS2_ARG_CONV_EXCL="*" tools/bazel' 'bazel' COMMAND ARGS)
+
+# Call sembuild (requires an internal repository checkout)
+[no-cd, no-exit-message]
+_sembuild *ARGS: (_run_in_semmle_code "./build" ARGS)

misc/just/codeql_test_run.py

@@ -0,0 +1,135 @@
+#!/usr/bin/env python3
+"""Run CodeQL tests with appropriate configuration.
+
+Called from just recipes as:
+    python3 codeql_test_run.py LANGUAGE BASE_FLAGS ALL_CHECKS_FLAGS EXTRA_ARGS
+"""
+
+import os
+import re
+import shlex
+import subprocess
+import sys
+from pathlib import Path
+
+JUST = os.environ.get("JUST_EXECUTABLE", "just")
+ERROR = os.environ.get("JUST_ERROR", "error: ")
+CMD_BEGIN = os.environ.get("CMD_BEGIN", "")
+CMD_END = os.environ.get("CMD_END", "")
+SEMMLE_CODE = os.environ.get("SEMMLE_CODE")
+
+
+def invoke(invocation, *, cwd=None, log_prefix=""):
+    prefix = f"{log_prefix} " if log_prefix else ""
+    print(f"{CMD_BEGIN}{prefix}{' '.join(invocation)}{CMD_END}")
+    try:
+        subprocess.run(invocation, check=True, cwd=cwd)
+    except subprocess.CalledProcessError as e:
+        return e.returncode
+    return 0
+
+
+def error(message):
+    print(f"{ERROR}{message}", file=sys.stderr)
+
+
+ENV_RE = re.compile(r"^[A-Z_][A-Z_0-9]*=.*$")
+
+
+def parse_args(args, argv_str):
+    """Parse a space-separated argument string into categorized arguments."""
+    for arg in shlex.split(argv_str, posix=sys.platform != "win32"):
+        if arg.startswith("--codeql="):
+            args["codeql"] = arg.split("=", 1)[1]
+        elif arg in ("+", "--all-checks"):
+            args["all"] = True
+        elif arg.startswith("-"):
+            args["flags"].append(arg)
+        elif ENV_RE.match(arg):
+            args["env"].append(arg)
+        elif arg:
+            args["tests"].append(arg)
+
+
+def main():
+    argv = sys.argv[1:]
+    if len(argv) < 4:
+        error(
+            "Usage: codeql_test_run.py LANGUAGE BASE_FLAGS ALL_CHECKS_FLAGS EXTRA_ARGS"
+        )
+        return 1
+
+    language, base_args, all_args, extra_args = argv[0], argv[1], argv[2], argv[3]
+    ram_per_thread = 3000 if sys.platform == "linux" else 2048
+    cpus = os.cpu_count() or 1
+
+    args = {
+        "tests": [],
+        "flags": [f"--ram={ram_per_thread * cpus}", f"-j{cpus}"],
+        "env": [],
+        "codeql": "build" if SEMMLE_CODE else "host",
+        "all": False,
+    }
+    parse_args(args, base_args)
+    parse_args(args, extra_args)
+    if args["all"]:
+        parse_args(args, all_args)
+
+    if not SEMMLE_CODE and args["codeql"] in ("build", "built"):
+        error(
+            "Using `--codeql=build` or `--codeql=built` requires working "
+            "with the internal repository"
+        )
+        return 1
+
+    if not args["tests"]:
+        args["tests"].append(".")
+
+    if args["codeql"] == "build":
+        if invoke([JUST, language, "build"], cwd=SEMMLE_CODE) != 0:
+            return 1
+
+    if args["codeql"] != "host":
+        # Disable the default implicit config file, but keep an explicit one.
+        # Same behavior wrt --codeql as the integration test runner.
+        os.environ.setdefault("CODEQL_CONFIG_FILE", ".")
+
+    for env_var in args["env"]:
+        key, _, value = env_var.partition("=")
+        if not key:
+            error(f"Invalid environment variable assignment: {env_var}")
+            return 1
+        os.environ[key] = value
+
+    # Resolve codeql executable
+    if args["codeql"] in ("built", "build"):
+        codeql = Path(SEMMLE_CODE, "target", "intree", f"codeql-{language}", "codeql")
+    elif args["codeql"] == "host":
+        codeql = Path("codeql")
+    else:
+        codeql = Path(args["codeql"])
+
+    if codeql.is_dir():
+        codeql = codeql / "codeql"
+
+    # On Windows, prefer codeql.exe over the Unix shell wrapper
+    if sys.platform == "win32" and codeql.suffix != ".exe":
+        exe = codeql.with_suffix(".exe")
+        if exe.exists():
+            codeql = exe
+
+    if args["codeql"] != "host" and not codeql.exists():
+        error(f"CodeQL executable not found: {codeql}")
+        return 1
+
+    return invoke(
+        [str(codeql), "test", "run", *args["flags"], "--", *args["tests"]],
+        log_prefix=" ".join(args["env"]),
+    )
+
+
+if __name__ == "__main__":
+    try:
+        sys.exit(main())
+    except KeyboardInterrupt:
+        sys.exit(128 + 2)

misc/just/defs.just

@@ -0,0 +1,57 @@
+import? '../../../semmle-code.just'  # internal repo just file, if present
+import 'semmle-code-stub.just'
+
+set fallback
+set allow-duplicate-recipes
+set allow-duplicate-variables
+
+export PATH_SEP := if os() == "windows" { ";" } else { ":" }
+export JUST_EXECUTABLE := just_executable()
+
+error := f'{{style("error")}}error{{NORMAL}}: '
+cmd_sep := "\n#--------------------------------------------------------\n"
+export CMD_BEGIN := style("command") + cmd_sep
+export CMD_END := cmd_sep + NORMAL
+export JUST_ERROR := error
+
+py := "python3"
+
+default_db_checks := """\
+    --check-databases \
+    --check-diff-informed \
+    --fail-on-trap-errors \
+"""
+
+[no-exit-message]
+@_require_semmle_code:
+    {{ if SEMMLE_CODE == "" { f'''
+        echo "{error} running this recipe requires doing so from an internal repository checkout" >&2
+        exit 1
+    ''' } else { "" } }}
+
+[no-cd]
+_run +ARGS:
+    {{ cmd_sep }}{{ ARGS }}{{ cmd_sep }}
+
+[no-cd]
+_run_in DIR +ARGS:
+    {{ cmd_sep }}cd "{{ DIR }}"; {{ ARGS }}{{ cmd_sep }}
+
+[no-cd]
+_run_in_semmle_code +ARGS: _require_semmle_code (_run_in "$SEMMLE_CODE" ARGS)
+
+[no-cd, positional-arguments, no-exit-message]
+@_just +ARGS:
+    echo "-> just $@"
+    "{{ JUST_EXECUTABLE }}" "$@"
+
+[no-cd, positional-arguments]
+@_if_not_on_ci_just +ARGS:
+    if [ "${GITHUB_ACTIONS:-}" != "true" ]; then \
+        echo "-> just $@"; \
+        "$JUST_EXECUTABLE" "$@"; \
+    fi
+
+[no-cd, no-exit-message]
+_if_in_semmle_code THEN ELSE *ARGS:
+    {{ cmd_sep }}{{ if SEMMLE_CODE != "" { THEN } else { ELSE } }} {{ ARGS }}{{ cmd_sep }}

misc/just/format.just

@@ -0,0 +1,20 @@
+import "build.just"
+
+[no-cd, no-exit-message]
+_format_ql +ARGS: (_maybe_build_dist "nolang") (
+    _if_in_semmle_code
+        '"$SEMMLE_CODE/target/intree/codeql-nolang/codeql"'
+        'codeql'
+    (f"query format --in-place -v $(find {{ARGS}} -type f -name '*.ql' -or -name '*.qll')")
+)
+
+[no-cd, no-exit-message]
+_format_py *ARGS=".": (_if_in_semmle_code "uv run black" "black" ARGS)
+
+[no-cd, no-exit-message]
+_format_cpp *ARGS=".": (
+    _if_in_semmle_code
+        "uv run clang-format"
+        "clang-format"
+    (f"-i --verbose $(find {{ARGS}} -type f -name '*.h' -or -name '*.cpp')")
+)

misc/just/forward.just

@@ -0,0 +1,38 @@
+# Common verbs
+# See README.md in this directory for an overview.
+
+import "lib.just"
+
+
+# copy&paste necessary for each command until proper forwarding of multiple args is implemented
+# see https://github.com/casey/just/issues/1988
+
+_forward := f'{{py}} "{{source_dir()}}/forward_command.py"'
+
+alias t := test
+alias b := build
+alias g := generate
+alias gen := generate
+alias f := format
+alias l := lint
+
+[no-cd, positional-arguments, no-exit-message]
+@test *ARGS:
+    {{ _forward }} test "$@"
+
+
+[no-cd, positional-arguments, no-exit-message]
+@build *ARGS:
+    {{ _forward }} build "$@"
+
+[no-cd, positional-arguments, no-exit-message]
+@generate *ARGS:
+    {{ _forward }} generate "$@"
+
+[no-cd, positional-arguments, no-exit-message]
+@lint *ARGS:
+    {{ _forward }} lint "$@"
+
+[no-cd, positional-arguments, no-exit-message]
+@format *ARGS:
+    {{ _forward }} format "$@"

misc/just/forward_command.py

@@ -0,0 +1,113 @@
+#!/usr/bin/env python3
+"""Forward commands to language-specific justfiles.
+
+Called from just recipes as:
+    python3 forward_command.py COMMAND [ARGS...]
+"""
+
+import os
+import re
+import subprocess
+import sys
+from pathlib import Path
+
+JUST = os.environ.get("JUST_EXECUTABLE", "just")
+ERROR = os.environ.get("JUST_ERROR", "")
+
+
+def error(message):
+    print(f"{ERROR}{message}", file=sys.stderr)
+
+
+def get_just_context(justfile, cmd, flags, positional_args):
+    """Get the (cwd, args) for invoking just with the given justfile."""
+    if (
+        len(positional_args) == 1
+        and justfile == Path(positional_args[0]) / "justfile"
+    ):
+        # If there's only one positional argument and it matches the justfile
+        # path, suppress arguments so e.g. `just build ql/rust` becomes
+        # `just build` in the `ql/rust` directory
+        return positional_args[0], [cmd, *flags]
+    else:
+        return None, ["--justfile", str(justfile), cmd, *flags, *positional_args]
+
+
+def check_just_command(justfile, command, positional_args):
+    """Check if a justfile supports the given command."""
+    if not justfile.exists():
+        return False
+    cwd, args = get_just_context(justfile, command, [], positional_args)
+    result = subprocess.run(
+        [JUST, "--dry-run", *args],
+        cwd=cwd,
+        stdin=subprocess.DEVNULL,
+        stdout=subprocess.DEVNULL,
+        stderr=subprocess.PIPE,
+        text=True,
+    )
+    # Avoid having the forwarder find itself
+    return (
+        result.returncode == 0
+        and f'forward_command.py" {command} "$@"' not in result.stderr
+    )
+
+
+def find_justfile(command, arg):
+    """Search up the directory tree for a justfile supporting the command."""
+    for p in [Path(arg), *Path(arg).parents]:
+        candidate = p / "justfile"
+        if check_just_command(candidate, command, [arg]):
+            return candidate
+    return None
+
+
+def invoke_just(cwd, args):
+    """Run just with the given arguments."""
+    try:
+        subprocess.run([JUST, *args], check=True, cwd=cwd)
+    except subprocess.CalledProcessError as e:
+        return e.returncode
+    return 0
+
+
+def forward(cmd, args):
+    """Forward a command to language-specific justfiles."""
+    is_non_positional = re.compile(r"^(-.*|\+|[A-Z_][A-Z_0-9]*=.*)$")
+    flags = [arg for arg in args if is_non_positional.match(arg)]
+    positional_args = [arg for arg in args if not is_non_positional.match(arg)]
+
+    justfiles = {}
+    for arg in positional_args or ["."]:
+        justfile = find_justfile(cmd, arg)
+        if not justfile:
+            error(f"No justfile found for {cmd} on {arg}")
+            return 1
+        justfiles.setdefault(justfile, []).append(arg)
+
+    invocations = []
+    for justfile, pos_args in justfiles.items():
+        cwd, just_args = get_just_context(justfile, cmd, flags, pos_args)
+        prefix = f"cd {cwd}; " if cwd else ""
+        print(f"-> {prefix}just {' '.join(just_args)}")
+        invocations.append((cwd, just_args))
+
+    for cwd, just_args in invocations:
+        if invoke_just(cwd, just_args) != 0:
+            return 1
+    return 0
+
+
+def main():
+    argv = sys.argv[1:]
+    if not argv:
+        error("No command provided")
+        return 1
+    return forward(argv[0], argv[1:])
+
+
+if __name__ == "__main__":
+    try:
+        sys.exit(main())
+    except KeyboardInterrupt:
+        sys.exit(128 + 2)

misc/just/justfile

@@ -0,0 +1,2 @@
+format *ARGS=".":
+    npx prettier --write {{ ARGS }}

misc/just/language_tests.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python3
+"""Run all language tests for CI.
+
+Called from just recipes as:
+    python3 language_tests.py EXTRA_ARGS SOURCE_DIR ROOT1 [ROOT2 ...]
+"""
+
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+
+def main():
+    argv = sys.argv[1:]
+    if len(argv) < 3:
+        print(
+            "Usage: language_tests.py EXTRA_ARGS SOURCE_DIR ROOT1 [ROOT2 ...]",
+            file=sys.stderr,
+        )
+        return 1
+
+    extra_args, source_dir, *relative_roots = argv
+
+    semmle_code = Path(os.environ["SEMMLE_CODE"])
+    roots = [
+        os.path.relpath(Path(source_dir) / root, semmle_code)
+        for root in relative_roots
+    ]
+
+    just = os.environ.get("JUST_EXECUTABLE", "just")
+
+    # Find the nearest justfile at or above the first root
+    justfile_dir = Path(roots[0])
+    while not (justfile_dir / "justfile").exists():
+        parent = justfile_dir.parent
+        if parent == justfile_dir:
+            print(f"No justfile found above {roots[0]}", file=sys.stderr)
+            return 1
+        justfile_dir = parent
+
+    invocation = [
+        just,
+        "--justfile",
+        str(justfile_dir / "justfile"),
+        "test",
+        "--all-checks",
+        "--codeql=built",
+        *(a for a in extra_args.split(" ") if a),
+        *roots,
+    ]
+
+    print(f"-> just {' '.join(invocation[1:])}")
+    try:
+        subprocess.run(invocation, check=True, cwd=semmle_code)
+    except subprocess.CalledProcessError as e:
+        return e.returncode
+    return 0
+
+
+if __name__ == "__main__":
+    try:
+        sys.exit(main())
+    except KeyboardInterrupt:
+        sys.exit(128 + 2)

misc/just/lib.just

@@ -0,0 +1,47 @@
+# Helper recipes
+
+import "build.just"
+import "format.just"
+
+# Run language tests. `BASE_FLAGS` are used by default, `ALL_CHECK_FLAGS` are used if
+# `--all-checks` or `+` is passed along in `EXTRA_ARGS`
+[no-cd, positional-arguments, no-exit-message]
+@_codeql_test LANGUAGE BASE_FLAGS ALL_CHECKS_FLAGS EXTRA_ARGS:
+    {{ py }} "{{ source_dir() }}/codeql_test_run.py" "$@"
+
+# Run all language tests in ROOTS for a language. This is intended to be called by CI
+[no-cd, positional-arguments, no-exit-message]
+@_language_tests EXTRA_ARGS SOURCE_DIR +ROOTS: _require_semmle_code
+    {{ py }} "{{ source_dir() }}/language_tests.py" "$@"
+
+# Run integration tests. Requires an internal repository checkout
+[no-cd, no-exit-message]
+_integration_test *ARGS: _require_semmle_code (_run "$SEMMLE_CODE/tools/pytest" "--codeql=build-as-test" ARGS)
+
+# Generic run command recipe that can be used by other recipes, with nice rendering
+[no-cd]
+_run +ARGS:
+    {{ cmd_sep }}{{ ARGS }}{{ cmd_sep }}
+
+# Run a command in a specific directory
+[no-cd]
+_run_in DIR +ARGS:
+    {{ cmd_sep }}cd "{{ DIR }}"; {{ ARGS }}{{ cmd_sep }}
+
+# Run a command in the internal repository checkout
+[no-cd]
+_run_in_semmle_code +ARGS: _require_semmle_code (_run_in "$SEMMLE_CODE" ARGS)
+
+# Run a just recipe
+[no-cd, positional-arguments, no-exit-message]
+@_just +ARGS:
+    echo "-> just $@"
+    "{{ JUST_EXECUTABLE }}" "$@"
+
+# Run a just recipe, but only if we are not on CI
+[no-cd, positional-arguments]
+@_if_not_on_ci_just +ARGS:
+    if [ "${GITHUB_ACTIONS:-}" != "true" ]; then \
+        echo "-> just $@"; \
+        "$JUST_EXECUTABLE" "$@"; \
+    fi

misc/just/semmle-code-stub.just

@@ -0,0 +1 @@
+export SEMMLE_CODE := ""

python/justfile

@@ -0,0 +1,32 @@
+import '../lib.just'
+import 'ql/justfile'
+
+[group('build')]
+build: (_build_dist "python")
+
+# Long filename needed for extractor tests (too long for Git on Windows)
+[no-cd]
+@_ensure_long_filename:
+    #!/usr/bin/env bash
+    longfile="$SEMMLE_CODE/ql/python/ql/test/extractor-tests/long_path/really_rather_too_long_for_windows_path_length/with_unecessarily_longwinded_and_verbose_sub_folder/extremely_long_module_name_with_lots_of_digits_at_the_end_000000000000000000000000000000000000000000000000000000000000000000/test0000000000000000000000000000000000000000000000000000000.py"
+    mkdir -p "$(dirname "$longfile")"
+    touch "$longfile"
+
+[group('test')]
+language-tests-2 *EXTRA_ARGS: _ensure_long_filename (_language_tests (
+        _v2_env + ' ' + EXTRA_ARGS) source_dir()
+    'ql/test/library-tests'
+    'ql/test/query-tests'
+    'ql/test/extractor-tests'
+    'ql/test/experimental'
+    'ql/test/2')
+
+[group('test')]
+language-tests-3 *EXTRA_ARGS: _ensure_long_filename (_language_tests (
+        _v3_env + ' ' + EXTRA_ARGS) source_dir()
+    'ql/test/library-tests'
+    'ql/test/query-tests'
+    'ql/test/extractor-tests'
+    'ql/test/experimental'
+    'ql/test/modelling'
+    'ql/test/3')

python/ql/integration-tests/justfile

@@ -0,0 +1,4 @@
+import "../../../lib.just"
+
+[no-cd]
+test *ARGS=".": (_integration_test ARGS)

python/ql/justfile

@@ -0,0 +1,12 @@
+import "../../lib.just"
+
+[no-cd]
+format *ARGS=".": (_format_ql ARGS)
+
+consistency_queries := source_dir() / "consistency-queries"
+
+python_version := env("python_version", "3")
+
+_v2_env := "CODEQL_EXTRACTOR_PYTHON_ANALYSIS_VERSION=2 CODEQL_PYTHON_LEGACY_TEST_EXTRACTION_VERSION=2"
+_v3_env := "CODEQL_PYTHON_LEGACY_TEST_EXTRACTION_VERSION=3"
+_python_env := if python_version == "2" { _v2_env } else { _v3_env }

python/ql/test/justfile

@@ -0,0 +1,10 @@
+import "../justfile"
+
+base_flags := _python_env
+
+all_checks := f"""\
+    {{default_db_checks}}\
+    --consistency-queries={{consistency_queries}}"""
+
+[no-cd]
+test *ARGS=".": (_codeql_test "python" base_flags all_checks ARGS)

ruby/justfile

@@ -0,0 +1,7 @@
+import '../lib.just'
+
+[group('build')]
+build: (_build_dist "ruby")
+
+[group('test')]
+language-tests *EXTRA_ARGS: (_language_tests EXTRA_ARGS source_dir() 'ql/test')

ruby/ql/integration-tests/justfile

@@ -0,0 +1,4 @@
+import "../../../lib.just"
+
+[no-cd]
+test *ARGS=".": (_integration_test ARGS)

ruby/ql/justfile

@@ -0,0 +1,6 @@
+import "../../lib.just"
+
+[no-cd]
+format *ARGS=".": (_format_ql ARGS)
+
+consistency_queries := source_dir() / "consistency-queries"

ruby/ql/test/justfile

@@ -0,0 +1,15 @@
+import "../justfile"
+
+base_flags := ""
+
+all_checks := f"""\
+    {{default_db_checks}}\
+    --check-undefined-labels \
+    --check-unused-labels \
+    --check-repeated-labels \
+    --check-redefined-labels \
+    --check-use-before-definition \
+    --consistency-queries={{consistency_queries}}"""
+
+[no-cd]
+test *ARGS=".": (_codeql_test "ruby" base_flags all_checks ARGS)

rust/codegen/codegen.sh

@@ -2,7 +2,7 @@
 
 set -eu
 
-source misc/bazel/runfiles.sh 2>/dev/null || source external/ql+/misc/bazel/runfiles.sh
+source misc/bazel/runfiles.sh 2>/dev/null || source ../ql+/misc/bazel/runfiles.sh
 
 ast_generator="$(rlocation "$1")"
 grammar_file="$(rlocation "$2")"

rust/justfile

@@ -0,0 +1,14 @@
+import '../lib.just'
+
+install: (_bazel "run" "@codeql//rust:install")
+
+build: (_if_not_on_ci_just "generate" source_dir()) (_build_dist "rust")
+
+generate: (_bazel "run" "@codeql//rust/codegen")
+
+lint: (_run_in source_dir() "python3" "lint.py")
+
+format: (_run_in source_dir() "python3" "lint.py" "--format-only")
+
+[group('test')]
+language-tests *EXTRA_ARGS: (_language_tests EXTRA_ARGS source_dir() 'ql/test')

rust/lint.py

@@ -4,6 +4,14 @@ import subprocess
 import pathlib
 import shutil
 import sys
+import argparse
+
+
+def options():
+    parser = argparse.ArgumentParser(description="lint rust language pack code")
+    parser.add_argument("--format-only", action="store_true", help="Only apply formatting")
+    return parser.parse_args()
+
 
 
 def tool(name):
@@ -12,27 +20,35 @@ def tool(name):
     return ret
 
 
-this_dir = pathlib.Path(__file__).resolve().parent
+def main():
-
+    args = options()
-cargo = tool("cargo")
+    this_dir = pathlib.Path(__file__).resolve().parent
-bazel = tool("bazel")
-
-runs = []
 
 
-def run(tool, args, *, cwd=this_dir):
+    cargo = tool("cargo")
-    print("+", tool, args)
+    bazel = tool("bazel")
-    runs.append(subprocess.run([tool] + args.split(), cwd=cwd))
+
+    runs = []
 
 
-# make sure bazel-provided sources are put in tree for `cargo` to work with them
+    def run(tool, args, *, cwd=this_dir):
-run(bazel, "run ast-generator:inject-sources")
+        print("+", tool, args)
-run(cargo, "fmt --all --quiet")
+        runs.append(subprocess.run([tool] + args.split(), cwd=cwd))
 
-for manifest in this_dir.rglob("Cargo.toml"):
-    if not manifest.is_relative_to(this_dir / "ql") and not manifest.is_relative_to(this_dir / "integration-tests"):
-        run(cargo,
-            "clippy --fix --allow-dirty --allow-staged --quiet -- -D warnings",
-            cwd=manifest.parent)
 
-sys.exit(max(r.returncode for r in runs))
+    # make sure bazel-provided sources are put in tree for `cargo` to work with them
+    run(bazel, "run ast-generator:inject-sources")
+    run(cargo, "fmt --all --quiet")
+
+    if not args.format_only:
+        for manifest in this_dir.rglob("Cargo.toml"):
+            if not manifest.is_relative_to(this_dir / "ql") and not manifest.is_relative_to(this_dir / "integration-tests"):
+                run(cargo,
+                    "clippy --fix --allow-dirty --allow-staged --quiet -- -D warnings",
+                    cwd=manifest.parent)
+
+    return max(r.returncode for r in runs)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

rust/ql/integration-tests/justfile

@@ -0,0 +1,10 @@
+
+import "../../../lib.just"
+
+
+[no-cd]
+test *ARGS=".": (_if_not_on_ci_just "generate" source_dir()) (_integration_test ARGS)
+
+# TODO in separate PR
+# [no-cd]
+# format *ARGS=".": (_format_ql ARGS) (_format_py ARGS)

rust/ql/justfile

@@ -0,0 +1,6 @@
+import "../../lib.just"
+
+[no-cd]
+format *ARGS=".": (_format_ql ARGS)
+
+consistency_queries := source_dir() / "consistency-queries"

rust/ql/test/justfile

@@ -0,0 +1,8 @@
+import "../justfile"
+
+all_checks := f"""\
+    {{default_db_checks}}\
+     --consistency-queries={{consistency_queries}}"""
+
+[no-cd]
+test *ARGS=".": (_codeql_test "rust" "" all_checks ARGS)

swift/justfile

@@ -0,0 +1,17 @@
+import '../lib.just'
+import "../../ql/swift/ql/justfile"
+
+install: (_bazel "run" "@codeql//swift:install")
+
+[group('build')]
+build: (_build_dist "swift")
+
+generate: (_bazel "run" "@codeql//swift/codegen")
+
+format ARGS=".": (_format_cpp ARGS)
+
+[group('test')]
+language-tests *EXTRA_ARGS: (_language_tests EXTRA_ARGS source_dir() 'ql/test')
+
+[group('test')]
+extra-tests: (_sembuild "target/test/check-queries-swift") (_sembuild "target/test/check-db-upgrades-swift") (_sembuild "target/test/check-db-downgrades-swift")

swift/ql/integration-tests/justfile

@@ -0,0 +1,9 @@
+import "../../../lib.just"
+
+
+[no-cd]
+test *ARGS=".": (_just "generate") (_integration_test ARGS)
+
+# TODO in separate PR
+# [no-cd]
+# format *ARGS=".": (_format_ql ARGS) (_format_py ARGS)

swift/ql/justfile

@@ -0,0 +1,6 @@
+import "../../lib.just"
+
+[no-cd]
+format *ARGS=".": (_format_ql ARGS)
+
+consistency_queries := source_dir() / "consistency-queries"

swift/ql/test/justfile

@@ -0,0 +1,12 @@
+import "../justfile"
+
+all_checks := f"""\
+    {{default_db_checks}}\
+    --check-repeated-labels \
+    --check-redefined-labels \
+    --check-use-before-definition \
+    --consistency-queries={{consistency_queries}}"""
+
+
+[no-cd]
+test *ARGS=".": (_codeql_test "swift" "" all_checks ARGS)