Port Swift integration tests to pytest.

swift/ql/integration-tests/BUILD.bazel

@@ -1,7 +1,6 @@
 py_library(
     name = "utils",
     srcs = [
-        "create_database_utils.py",
         "diagnostics_test_utils.py",
     ],
     visibility = ["//swift:__subpackages__"],

swift/ql/integration-tests/create_database_utils.py

@@ -1,41 +0,0 @@
-"""
-Simplified version of internal `create_database_utils.py` used to run the tests locally, with
-minimal and swift-specialized functionality
-TODO unify integration testing code across the public and private repositories
-"""
-import subprocess
-import pathlib
-import sys
-import shutil
-
-
-def runSuccessfully(cmd):
-    res = subprocess.run(cmd)
-    if res.returncode:
-        print("FAILED", file=sys.stderr)
-        print(" ", *cmd, f"(exit code {res.returncode})", file=sys.stderr)
-        sys.exit(res.returncode)
-
-def runUnsuccessfully(cmd):
-    res = subprocess.run(cmd)
-    if res.returncode == 0:
-        print("FAILED", file=sys.stderr)
-        print(" ", *cmd, f"(exit code 0, expected to fail)", file=sys.stderr)
-        sys.exit(1)
-
-
-def run_codeql_database_create(cmds, lang, keep_trap=True, db=None, runFunction=runSuccessfully):
-    """ db parameter is here solely for compatibility with the internal test runner """
-    assert lang == 'swift'
-    codeql_root = pathlib.Path(__file__).parents[3]
-    shutil.rmtree("test-db", ignore_errors=True)
-    cmd = [
-        "codeql", "database", "create",
-        "-s", ".", "-l", "swift", f"--search-path={codeql_root}", "--no-cleanup",
-    ]
-    if keep_trap:
-        cmd.append("--keep-trap")
-    for c in cmds:
-        cmd += ["-c", c]
-    cmd.append("test-db")
-    runFunction(cmd)

swift/ql/integration-tests/legacy

@@ -1 +0,0 @@
-These tests are still run with the legacy test runner

swift/ql/integration-tests/linux-only/RegexLiteralExpr/test.py

@@ -1,5 +1,6 @@
-from create_database_utils import *
+import runs_on
 
-run_codeql_database_create([
-    "swiftc -enable-bare-slash-regex regex.swift -o /dev/null",
-], lang="swift")
+
+@runs_on.linux
+def test(codeql, swift):
+    codeql.database.create(command="swiftc -enable-bare-slash-regex regex.swift -o /dev/null")

swift/ql/integration-tests/linux-only/autobuilder/unsupported-os/test.py

@@ -1,5 +1,6 @@
-from create_database_utils import *
-from diagnostics_test_utils import *
+import runs_on
 
-run_codeql_database_create([], lang='swift', keep_trap=True, db=None, runFunction=runUnsuccessfully)
-check_diagnostics()
+
+@runs_on.linux
+def test(codeql, swift):
+    codeql.database.create(_assert_failure=True)

swift/ql/integration-tests/osx-only/autobuilder/failure/test.py

@@ -1,5 +1,6 @@
-from create_database_utils import *
-from diagnostics_test_utils import *
+import runs_on
 
-run_codeql_database_create([], lang='swift', keep_trap=True, db=None, runFunction=runUnsuccessfully)
-check_diagnostics()
+
+@runs_on.macos
+def test(codeql, swift):
+    codeql.database.create(_assert_failure=True)

swift/ql/integration-tests/osx-only/autobuilder/no-build-system/test.py

@@ -1,5 +1,6 @@
-from create_database_utils import *
-from diagnostics_test_utils import *
+import runs_on
 
-run_codeql_database_create([], lang='swift', keep_trap=True, db=None, runFunction=runUnsuccessfully)
-check_diagnostics()
+
+@runs_on.macos
+def test(codeql, swift):
+    codeql.database.create(_assert_failure=True)

swift/ql/integration-tests/osx-only/autobuilder/no-swift-with-spm/test.py

@@ -1,5 +1,6 @@
-from create_database_utils import *
-from diagnostics_test_utils import *
+import runs_on
 
-run_codeql_database_create([], lang='swift', keep_trap=True, db=None, runFunction=runUnsuccessfully)
-check_diagnostics()
+
+@runs_on.macos
+def test(codeql, swift):
+    codeql.database.create(_assert_failure=True)

swift/ql/integration-tests/osx-only/autobuilder/no-swift/test.py

@@ -1,5 +1,6 @@
-from create_database_utils import *
-from diagnostics_test_utils import *
+import runs_on
 
-run_codeql_database_create([], lang='swift', keep_trap=True, db=None, runFunction=runUnsuccessfully)
-check_diagnostics()
+
+@runs_on.macos
+def test(codeql, swift):
+    codeql.database.create(_assert_failure=True)

swift/ql/integration-tests/osx-only/autobuilder/no-xcode-with-spm/test.py

@@ -1,5 +1,6 @@
-from create_database_utils import *
-from diagnostics_test_utils import *
+import runs_on
 
-run_codeql_database_create([], lang='swift', keep_trap=True, db=None, runFunction=runUnsuccessfully)
-check_diagnostics()
+
+@runs_on.macos
+def test(codeql, swift):
+    codeql.database.create(_assert_failure=True)

swift/ql/integration-tests/osx-only/autobuilder/only-tests-with-spm/test.py

@@ -1,5 +1,6 @@
-from create_database_utils import *
-from diagnostics_test_utils import *
+import runs_on
 
-run_codeql_database_create([], lang='swift', keep_trap=True, db=None, runFunction=runUnsuccessfully)
-check_diagnostics()
+
+@runs_on.macos
+def test(codeql, swift):
+    codeql.database.create(_assert_failure=True)

swift/ql/integration-tests/osx-only/autobuilder/only-tests/test.py

@@ -1,5 +1,6 @@
-from create_database_utils import *
-from diagnostics_test_utils import *
+import runs_on
 
-run_codeql_database_create([], lang='swift', keep_trap=True, db=None, runFunction=runUnsuccessfully)
-check_diagnostics()
+
+@runs_on.macos
+def test(codeql, swift):
+    codeql.database.create(_assert_failure=True)

swift/ql/integration-tests/osx-only/autobuilder/xcode-fails-spm-works/test.py

@@ -1,5 +1,8 @@
-from create_database_utils import *
+import runs_on
+import pytest
 
-runSuccessfully(['xcodebuild', 'clean'])
-runSuccessfully(['swift', 'package', 'clean'])
-run_codeql_database_create([], lang='swift', keep_trap=True)
+
+@runs_on.macos
+@pytest.mark.ql_test("DB-CHECK", xfail=True)
+def test(codeql, swift):
+    codeql.database.create()

swift/ql/integration-tests/osx-only/canonical-case/test.py

@@ -1,5 +1,6 @@
-from create_database_utils import *
+import runs_on
 
-run_codeql_database_create([
-    './build.sh',
-], lang='swift')
+
+@runs_on.macos
+def test(codeql, swift):
+    codeql.database.create(command="./build.sh")

swift/ql/integration-tests/osx-only/hello-xcode/test.py

@@ -1,9 +1,13 @@
-from create_database_utils import *
+import runs_on
+import pytest
 
-run_codeql_database_create([
-    'xcodebuild clean',
-    'xcodebuild build '
-    '-project codeql-swift-autobuild-test.xcodeproj '
-    '-target codeql-swift-autobuild-test '
-    'CODE_SIGNING_REQUIRED=NO CODE_SIGNING_ALLOWED=NO',
-], lang='swift', keep_trap=True)
+
+@runs_on.macos
+@pytest.mark.ql_test("DB-CHECK", xfail=True)
+def test(codeql, swift):
+    codeql.database.create(
+        command="xcodebuild build "
+        "-project codeql-swift-autobuild-test.xcodeproj "
+        "-target codeql-swift-autobuild-test "
+        "CODE_SIGNING_REQUIRED=NO CODE_SIGNING_ALLOWED=NO",
+    )

swift/ql/integration-tests/posix-only/cross-references/test.py

@@ -1,6 +1,8 @@
-from create_database_utils import *
+import runs_on
+import pytest
 
-run_codeql_database_create([
-    'swift package clean',
-    'swift build'
-], lang='swift')
+
+@runs_on.posix
+@pytest.mark.ql_test("DB-CHECK", xfail=True)
+def test(codeql, swift):
+    codeql.database.create(command="swift build")

swift/ql/integration-tests/posix-only/deduplication/test.py

@@ -1,6 +1,6 @@
-from create_database_utils import *
+import runs_on
 
-run_codeql_database_create([
-    'swift package clean',
-    'swift build',
-], lang='swift', keep_trap=True)
+
+@runs_on.posix
+def test(codeql, swift):
+    codeql.database.create(command="swift build", keep_trap=True)

swift/ql/integration-tests/posix-only/frontend-invocations/test.py

@@ -1,22 +1,24 @@
-from create_database_utils import *
-from subprocess import check_call
 from hashlib import sha256
 from pathlib import Path
+import runs_on
+import pytest
 
-run_codeql_database_create([
-    './build.sh',
-], lang='swift',
-)
 
-with open('hashes.expected', 'w') as expected:
-    for f in sorted(Path().glob("*.swiftmodule")):
-        with open(f, 'rb') as module:
-            print(f.name, sha256(module.read()).hexdigest(), file=expected)
+@runs_on.posix
+@pytest.mark.ql_test("DB-CHECK", xfail=True)
+def test(codeql, swift, expected_files):
+    codeql.database.create(command="./build.sh")
+    with open("hashes.expected", "w") as expected:
+        for f in sorted(Path().glob("*.swiftmodule")):
+            with open(f, "rb") as module:
+                print(f.name, sha256(module.read()).hexdigest(), file=expected)
 
-with open('hashes.actual', 'w') as actual:
-    hashes = [(s.name, s.resolve().name) for s in Path("test-db/working/swift-extraction-artifacts/store").iterdir()]
-    hashes.sort()
-    for module, hash in hashes:
-        print(module, hash, file=actual)
-
-check_call(['diff', '-u', 'hashes.expected', 'hashes.actual'])
+    with open("hashes.actual", "w") as actual:
+        hashes = [
+            (s.name, s.resolve().name)
+            for s in Path("test-db/working/swift-extraction-artifacts/store").iterdir()
+        ]
+        hashes.sort()
+        for module, hash in hashes:
+            print(module, hash, file=actual)
+    expected_files.add("hashes.expected")

swift/ql/integration-tests/posix-only/hello-world/test.py

@@ -1,6 +1,6 @@
-from create_database_utils import *
+import runs_on
 
-run_codeql_database_create([
-    'swift package clean',
-    'swift build'
-], lang='swift')
+
+@runs_on.posix
+def test(codeql, swift):
+    codeql.database.create(command="swift build")

swift/ql/integration-tests/posix-only/linkage-awareness/test.py

@@ -1,5 +1,8 @@
-from create_database_utils import *
+import runs_on
+import pytest
 
-run_codeql_database_create([
-    './build.sh',
-], lang='swift', keep_trap=True)
+
+@runs_on.posix
+@pytest.mark.ql_test("DB-CHECK", xfail=True)
+def test(codeql, swift):
+    codeql.database.create(command="./build.sh")

swift/ql/integration-tests/posix-only/partial-modules/test.py

@@ -1,6 +1,8 @@
-from create_database_utils import *
+import runs_on
+import pytest
 
-run_codeql_database_create([
-    'swift package clean',
-    'swift build'
-], lang='swift', keep_trap=True)
+
+@runs_on.posix
+@pytest.mark.ql_test("DB-CHECK", xfail=True)
+def test(codeql, swift):
+    codeql.database.create(command="swift build")

swift/ql/integration-tests/posix-only/symlinks/test.py

@@ -1,20 +1,21 @@
-from create_database_utils import *
 import os
+import runs_on
+import pytest
 
-symlinks = ['preserve/Sources/A.swift', 'resolve/Sources/A.swift']
 
-for s in symlinks:
-    try:
-        os.symlink(os.getcwd() + '/main.swift', s)
-    except:
-        pass
+@runs_on.posix
+@pytest.mark.ql_test("DB-CHECK", xfail=True)
+def test(codeql, swift):
+    symlinks = ["preserve/Sources/A.swift", "resolve/Sources/A.swift"]
 
-run_codeql_database_create([
-    'swift package clean --package-path resolve',
-    'swift build --package-path resolve',
-    'swift package clean --package-path preserve',
-    'env CODEQL_PRESERVE_SYMLINKS=true swift build --package-path preserve'
-], lang='swift', keep_trap=True)
-
-for s in symlinks:
-    os.unlink(s)
+    for s in symlinks:
+        try:
+            os.symlink(os.getcwd() + "/main.swift", s)
+        except:
+            pass
+    codeql.database.create(
+        command=[
+            "swift build --package-path resolve",
+            "env CODEQL_PRESERVE_SYMLINKS=true swift build --package-path preserve",
+        ]
+    )

swift/ql/integration-tests/qlpack.yml

@@ -1,4 +0,0 @@
-dependencies:
-  codeql/swift-all: '*'
-  codeql/swift-queries: '*'
-warnOnImplicitThis: true

swift/ql/integration-tests/runner.py

@@ -1,90 +0,0 @@
-#!/bin/env python3
-"""
-recreation of internal `integration-tests-runner.py` to run the tests locally, with minimal
-and swift-specialized functionality.
-
-This runner requires:
-* a codeql CLI binary in PATH
-* `bazel run //swift:create_extractor_pack` to have been run
-"""
-
-import pathlib
-import os
-import sys
-import subprocess
-import argparse
-import shutil
-import platform
-
-this_dir = pathlib.Path(__file__).parent.resolve()
-codeql_root = this_dir.parents[2]
-
-def options():
-    p = argparse.ArgumentParser()
-    p.add_argument("--test-dir", "-d", type=pathlib.Path, action="append")
-    # FIXME: the following should be the default
-    p.add_argument("--check-databases", action="store_true")
-    p.add_argument("--learn", action="store_true")
-    p.add_argument("--threads", "-j", type=int, default=0)
-    p.add_argument("--compilation-cache")
-    return p.parse_args()
-
-
-def execute_test(path):
-    shutil.rmtree(path.parent / "test-db", ignore_errors=True)
-    return subprocess.run([sys.executable, "-u", path.name], cwd=path.parent).returncode == 0
-
-def skipped(test):
-    if platform.system() == "Darwin":
-        return "linux-only" in test.parts
-    else:
-        return "osx-only" in test.parts
-
-
-def main(opts):
-    test_dirs = opts.test_dir or [this_dir]
-    tests = [t for d in test_dirs for t in d.rglob("test.py") if not skipped(t)]
-    if opts.learn:
-        os.environ["CODEQL_INTEGRATION_TEST_LEARN"] = "true"
-
-    if not tests:
-        print("No tests found", file=sys.stderr)
-        return False
-
-    os.environ["PYTHONPATH"] = str(this_dir)
-    os.environ["CODEQL_CONFIG_FILE"] = "/dev/null"
-    failed_db_creation = []
-    succesful_db_creation = []
-    for t in tests:
-        (succesful_db_creation if execute_test(t) else failed_db_creation).append(t)
-
-    if succesful_db_creation:
-        cmd = [
-            "codeql", "test", "run",
-            f"--additional-packs={codeql_root}",
-            "--keep-databases",
-            "--dataset=test-db/db-swift",
-            f"--threads={opts.threads}",
-        ]
-        if opts.check_databases:
-            cmd.append("--check-databases")
-        else:
-            cmd.append("--no-check-databases")
-        if opts.learn:
-            cmd.append("--learn")
-        if opts.compilation_cache:
-            cmd.append(f'--compilation-cache="{opts.compilation_cache}"')
-        cmd.extend(str(t.parent) for t in succesful_db_creation)
-        ql_test_success = subprocess.run(cmd).returncode == 0
-
-    if failed_db_creation:
-        print("Database creation failed:", file=sys.stderr)
-        for t in failed_db_creation:
-            print(" ", t.parent, file=sys.stderr)
-        return False
-
-    return ql_test_success
-
-
-if __name__ == "__main__":
-    sys.exit(0 if main(options()) else 1)