diff --git a/.github/workflows/swift-qltest.yml b/.github/workflows/swift-qltest.yml index d50e1748de0..b86c876aab5 100644 --- a/.github/workflows/swift-qltest.yml +++ b/.github/workflows/swift-qltest.yml @@ -23,12 +23,23 @@ jobs: - uses: ./.github/actions/fetch-codeql - name: Check QL formatting run: find ql "(" -name "*.ql" -or -name "*.qll" ")" -print0 | xargs -0 codeql query format --check-only + qltest-test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: bazelbuild/setup-bazelisk@v2 + - uses: actions/setup-python@v4 + with: + python-version-file: 'swift/.python-version' + - name: Test qltest.sh + run: | + bazel test //swift/tools/test/qltest qltest: runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - os : [ubuntu-20.04, macos-latest] + os: [ ubuntu-20.04, macos-latest ] steps: - uses: actions/checkout@v3 - uses: ./.github/actions/fetch-codeql diff --git a/swift/tools/create_extractor_pack.py b/swift/create_extractor_pack.py similarity index 100% rename from swift/tools/create_extractor_pack.py rename to swift/create_extractor_pack.py diff --git a/swift/tools/BUILD.bazel b/swift/tools/BUILD.bazel new file mode 100644 index 00000000000..165bae13629 --- /dev/null +++ b/swift/tools/BUILD.bazel @@ -0,0 +1,23 @@ +package(default_visibility = ["//visibility:public"]) + +load("@rules_pkg//:mappings.bzl", "pkg_attributes", "pkg_files") + +sh_binary( + name = "qltest", + srcs = ["qltest.sh"], +) + +sh_binary( + name = "autobuild", + srcs = ["autobuild.sh"], +) + +pkg_files( + name = "tools", + srcs = [ + ":autobuild", + ":qltest", + ], + attributes = pkg_attributes(mode = "0755"), + prefix = "tools", +) diff --git a/swift/tools/test/qltest/BUILD.bazel b/swift/tools/test/qltest/BUILD.bazel new file mode 100644 index 00000000000..f16563eb21d --- /dev/null +++ b/swift/tools/test/qltest/BUILD.bazel @@ -0,0 +1,25 @@ +py_library( + name = "utils", + srcs = ["utils.py"], +) + +[ + py_test( + name = "test_%s" % test[:test.find("/")], + size = "small", + srcs = [test], + args = [ + "$(location //swift/tools:qltest)", + ], + data = [ + "//swift/tools:qltest", + ] + glob([test.replace("test.py", "*")]), + main = test, + deps = [":utils"], + ) + for test in glob(["*/test.py"]) +] + +test_suite( + name = "qltest", +) diff --git a/swift/tools/test/qltest/extractor_options/a.swift b/swift/tools/test/qltest/extractor_options/a.swift new file mode 100644 index 00000000000..027cd50d553 --- /dev/null +++ b/swift/tools/test/qltest/extractor_options/a.swift @@ -0,0 +1 @@ +//codeql-extractor-options: -some -option-for-a diff --git a/swift/tools/test/qltest/extractor_options/b.swift b/swift/tools/test/qltest/extractor_options/b.swift new file mode 100644 index 00000000000..3625c73b22b --- /dev/null +++ b/swift/tools/test/qltest/extractor_options/b.swift @@ -0,0 +1 @@ +//codeql-extractor-options: -some-other -option-for-b diff --git a/swift/tools/test/qltest/extractor_options/test.py b/swift/tools/test/qltest/extractor_options/test.py new file mode 100644 index 00000000000..d3ab71f9252 --- /dev/null +++ b/swift/tools/test/qltest/extractor_options/test.py @@ -0,0 +1,8 @@ +from swift.tools.test.qltest.utils import * + +set_dummy_extractor() +run_qltest() +assert_extractor_executed_with( + "a.swift -some -option-for-a", + "b.swift -some-other -option-for-b", +) diff --git a/swift/tools/test/qltest/failing_run/a.swift b/swift/tools/test/qltest/failing_run/a.swift new file mode 100644 index 00000000000..e69de29bb2d diff --git a/swift/tools/test/qltest/failing_run/b.swift b/swift/tools/test/qltest/failing_run/b.swift new file mode 100644 index 00000000000..e69de29bb2d diff --git a/swift/tools/test/qltest/failing_run/c.swift b/swift/tools/test/qltest/failing_run/c.swift new file mode 100644 index 00000000000..e69de29bb2d diff --git a/swift/tools/test/qltest/failing_run/test.py b/swift/tools/test/qltest/failing_run/test.py new file mode 100644 index 00000000000..e0d8514dc21 --- /dev/null +++ b/swift/tools/test/qltest/failing_run/test.py @@ -0,0 +1,9 @@ +from swift.tools.test.qltest.utils import * + +set_dummy_extractor('if [[ " $@ " =~ b.swift ]]; then exit 1; fi') +run_qltest(expected_returncode=1) +assert_extractor_executed_with( + "a.swift", + "b.swift", + "c.swift", +) diff --git a/swift/tools/test/qltest/normal_run/a.swift b/swift/tools/test/qltest/normal_run/a.swift new file mode 100644 index 00000000000..e69de29bb2d diff --git a/swift/tools/test/qltest/normal_run/b.swift b/swift/tools/test/qltest/normal_run/b.swift new file mode 100644 index 00000000000..e69de29bb2d diff --git a/swift/tools/test/qltest/normal_run/c.swift b/swift/tools/test/qltest/normal_run/c.swift new file mode 100644 index 00000000000..e69de29bb2d diff --git a/swift/tools/test/qltest/normal_run/test.py b/swift/tools/test/qltest/normal_run/test.py new file mode 100644 index 00000000000..8825540d471 --- /dev/null +++ b/swift/tools/test/qltest/normal_run/test.py @@ -0,0 +1,9 @@ +from swift.tools.test.qltest.utils import * + +set_dummy_extractor() +run_qltest() +assert_extractor_executed_with( + "a.swift", + "b.swift", + "c.swift", +) diff --git a/swift/tools/test/qltest/utils.py b/swift/tools/test/qltest/utils.py new file mode 100644 index 00000000000..49ee86bf2f0 --- /dev/null +++ b/swift/tools/test/qltest/utils.py @@ -0,0 +1,68 @@ +import sys +import pathlib +import subprocess +import os +import itertools +import inspect + + +def _absolute_path(*path: str) -> str: + return str(pathlib.Path(*path).absolute()) + + +qltest = pathlib.Path(sys.argv[1]).absolute() +script_dir = pathlib.Path(__file__).parent.absolute() +execution_log = pathlib.Path("execution.log").absolute() + +swift_root = "dummy_root" +platform = "dummy_plat" + +def _get_test_dir(): + frame = inspect.stack()[2] + module = inspect.getmodule(frame[0]) + return pathlib.Path(module.__file__).parent + +def set_dummy_extractor(*cmds): + extractor = _get_test_dir() / swift_root / "tools" / platform / "extractor" + extractor.parent.mkdir(parents=True, exist_ok=True) + execution_log.unlink(missing_ok=True) + with open(extractor, "w") as extractor_out: + print("#!/bin/bash", file=extractor_out) + print(f'echo "$@" >> {execution_log}', file=extractor_out) + for cmd in cmds: + print(cmd, file=extractor_out) + os.chmod(extractor, 0o777) + + +def run_qltest(expected_returncode=0): + current_dir = _absolute_path() + test_dir = _get_test_dir() + + env = { + "CODEQL_EXTRACTOR_SWIFT_LOG_DIR": ".", + "CODEQL_EXTRACTOR_SWIFT_ROOT": swift_root, + "CODEQL_PLATFORM": platform, + "CODEQL_EXTRACTOR_SWIFT_TRAP_DIR": "traps", + } + + qltest_returncode = subprocess.run([str(qltest)], env=env, cwd=str(test_dir), + stdout=subprocess.DEVNULL).returncode + + if qltest_returncode != expected_returncode: + print(f"qltest returned with exit status {qltest_returncode}, expecting {expected_returncode}") + with open(test_dir / "qltest.log", "r") as log: + sys.stdout.write(log.read()) + sys.exit(1) + + +def assert_extractor_executed_with(*flags): + with open(execution_log) as execution: + for actual, expected in itertools.zip_longest(execution, flags): + if actual: + actual = actual.strip() + expected_prefix = f"-sdk {swift_root}/qltest/{platform}/sdk -c -primary-file " + assert actual.startswith(expected_prefix), f"correct sdk option not found in\n{actual}" + actual = actual[len(expected_prefix):] + assert actual, f"\nnot encountered: {expected}" + assert expected, f"\nunexpected: {actual}" + assert actual == expected, f"\nexpecting: {actual}\ngot: {expected}"