Swift: add qltest.sh tests

.github/workflows/swift-qltest.yml

@@ -23,12 +23,23 @@ jobs:
       - uses: ./.github/actions/fetch-codeql
       - name: Check QL formatting
         run: find ql "(" -name "*.ql" -or -name "*.qll" ")" -print0 | xargs -0 codeql query format --check-only
+  qltest-test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: bazelbuild/setup-bazelisk@v2
+      - uses: actions/setup-python@v4
+        with:
+          python-version-file: 'swift/.python-version'
+      - name: Test qltest.sh
+        run: |
+          bazel test //swift/tools/test/qltest
   qltest:
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        os : [ubuntu-20.04, macos-latest]
+        os: [ ubuntu-20.04, macos-latest ]
     steps:
       - uses: actions/checkout@v3
       - uses: ./.github/actions/fetch-codeql

swift/tools/BUILD.bazel

@@ -0,0 +1,23 @@
+package(default_visibility = ["//visibility:public"])
+
+load("@rules_pkg//:mappings.bzl", "pkg_attributes", "pkg_files")
+
+sh_binary(
+    name = "qltest",
+    srcs = ["qltest.sh"],
+)
+
+sh_binary(
+    name = "autobuild",
+    srcs = ["autobuild.sh"],
+)
+
+pkg_files(
+    name = "tools",
+    srcs = [
+        ":autobuild",
+        ":qltest",
+    ],
+    attributes = pkg_attributes(mode = "0755"),
+    prefix = "tools",
+)

swift/tools/test/qltest/BUILD.bazel

@@ -0,0 +1,25 @@
+py_library(
+    name = "utils",
+    srcs = ["utils.py"],
+)
+
+[
+    py_test(
+        name = "test_%s" % test[:test.find("/")],
+        size = "small",
+        srcs = [test],
+        args = [
+            "$(location //swift/tools:qltest)",
+        ],
+        data = [
+            "//swift/tools:qltest",
+        ] + glob([test.replace("test.py", "*")]),
+        main = test,
+        deps = [":utils"],
+    )
+    for test in glob(["*/test.py"])
+]
+
+test_suite(
+    name = "qltest",
+)

swift/tools/test/qltest/extractor_options/a.swift

@@ -0,0 +1 @@
+//codeql-extractor-options: -some -option-for-a

swift/tools/test/qltest/extractor_options/b.swift

@@ -0,0 +1 @@
+//codeql-extractor-options: -some-other -option-for-b

swift/tools/test/qltest/extractor_options/test.py

@@ -0,0 +1,8 @@
+from swift.tools.test.qltest.utils import *
+
+set_dummy_extractor()
+run_qltest()
+assert_extractor_executed_with(
+    "a.swift -some -option-for-a",
+    "b.swift -some-other -option-for-b",
+)

swift/tools/test/qltest/failing_run/test.py

@@ -0,0 +1,9 @@
+from swift.tools.test.qltest.utils import *
+
+set_dummy_extractor('if [[ " $@ " =~ b.swift ]]; then exit 1; fi')
+run_qltest(expected_returncode=1)
+assert_extractor_executed_with(
+    "a.swift",
+    "b.swift",
+    "c.swift",
+)

swift/tools/test/qltest/normal_run/test.py

@@ -0,0 +1,9 @@
+from swift.tools.test.qltest.utils import *
+
+set_dummy_extractor()
+run_qltest()
+assert_extractor_executed_with(
+    "a.swift",
+    "b.swift",
+    "c.swift",
+)

swift/tools/test/qltest/utils.py

@@ -0,0 +1,68 @@
+import sys
+import pathlib
+import subprocess
+import os
+import itertools
+import inspect
+
+
+def _absolute_path(*path: str) -> str:
+    return str(pathlib.Path(*path).absolute())
+
+
+qltest = pathlib.Path(sys.argv[1]).absolute()
+script_dir = pathlib.Path(__file__).parent.absolute()
+execution_log = pathlib.Path("execution.log").absolute()
+
+swift_root = "dummy_root"
+platform = "dummy_plat"
+
+def _get_test_dir():
+    frame = inspect.stack()[2]
+    module = inspect.getmodule(frame[0])
+    return pathlib.Path(module.__file__).parent
+
+def set_dummy_extractor(*cmds):
+    extractor = _get_test_dir() / swift_root / "tools" / platform / "extractor"
+    extractor.parent.mkdir(parents=True, exist_ok=True)
+    execution_log.unlink(missing_ok=True)
+    with open(extractor, "w") as extractor_out:
+        print("#!/bin/bash", file=extractor_out)
+        print(f'echo "$@" >> {execution_log}', file=extractor_out)
+        for cmd in cmds:
+            print(cmd, file=extractor_out)
+    os.chmod(extractor, 0o777)
+
+
+def run_qltest(expected_returncode=0):
+    current_dir = _absolute_path()
+    test_dir = _get_test_dir()
+
+    env = {
+        "CODEQL_EXTRACTOR_SWIFT_LOG_DIR": ".",
+        "CODEQL_EXTRACTOR_SWIFT_ROOT": swift_root,
+        "CODEQL_PLATFORM": platform,
+        "CODEQL_EXTRACTOR_SWIFT_TRAP_DIR": "traps",
+    }
+
+    qltest_returncode = subprocess.run([str(qltest)], env=env, cwd=str(test_dir),
+                                       stdout=subprocess.DEVNULL).returncode
+
+    if qltest_returncode != expected_returncode:
+        print(f"qltest returned with exit status {qltest_returncode}, expecting {expected_returncode}")
+        with open(test_dir / "qltest.log", "r") as log:
+            sys.stdout.write(log.read())
+        sys.exit(1)
+
+
+def assert_extractor_executed_with(*flags):
+    with open(execution_log) as execution:
+        for actual, expected in itertools.zip_longest(execution, flags):
+            if actual:
+                actual = actual.strip()
+                expected_prefix = f"-sdk {swift_root}/qltest/{platform}/sdk -c -primary-file "
+                assert actual.startswith(expected_prefix), f"correct sdk option not found in\n{actual}"
+                actual = actual[len(expected_prefix):]
+            assert actual, f"\nnot encountered: {expected}"
+            assert expected, f"\nunexpected: {actual}"
+            assert actual == expected, f"\nexpecting: {actual}\ngot:       {expected}"