Move swift/codegen to misc/codegen

.github/workflows/swift.yml

@@ -5,6 +5,7 @@ on:
     paths:
       - "swift/**"
       - "misc/bazel/**"
+      - "misc/codegen/**"
       - "*.bazel*"
       - .github/workflows/swift.yml
       - .github/actions/**
@@ -19,6 +20,7 @@ on:
     paths:
       - "swift/**"
       - "misc/bazel/**"
+      - "misc/codegen/**"
       - "*.bazel*"
       - .github/workflows/swift.yml
       - .github/actions/**

.pre-commit-config.yaml

@@ -53,5 +53,5 @@ repos:
         name: Run Swift code generation unit tests
         files: ^swift/codegen/.*\.py$
         language: system
-        entry: bazel test //swift/codegen/test
+        entry: bazel test //misc/codegen/test
         pass_filenames: false

CODEOWNERS

@@ -6,6 +6,7 @@
 /python/ @github/codeql-dynamic
 /ruby/ @github/codeql-dynamic
 /swift/ @github/codeql-swift
+/misc/codegen/ @github/codeql-swift
 /java/kotlin-extractor/ @github/codeql-kotlin
 /java/kotlin-explorer/ @github/codeql-kotlin
 

misc/bazel/workspace.bzl

@@ -33,3 +33,13 @@ def codeql_workspace(repository_name = "codeql"):
             "https://github.com/bazelbuild/rules_python/archive/refs/tags/0.8.1.tar.gz",
         ],
     )
+
+    maybe(
+        repo_rule = http_archive,
+        name = "bazel_skylib",
+        sha256 = "b8a1527901774180afc798aeb28c4634bdccf19c4d98e7bdd1ce79d1fe9aaad7",
+        urls = [
+            "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.4.1/bazel-skylib-1.4.1.tar.gz",
+            "https://github.com/bazelbuild/bazel-skylib/releases/download/1.4.1/bazel-skylib-1.4.1.tar.gz",
+        ],
+    )

misc/bazel/workspace_deps.bzl

@@ -1,9 +1,11 @@
 load("@rules_pkg//:deps.bzl", "rules_pkg_dependencies")
 load("@rules_python//python:pip.bzl", "pip_install")
+load("@bazel_skylib//:workspace.bzl", "bazel_skylib_workspace")
 
 def codeql_workspace_deps(repository_name = "codeql"):
     pip_install(
-       name = "swift_codegen_deps",
+       name = "codegen_deps",
-       requirements = "@%s//swift/codegen:requirements.txt" % repository_name,
+       requirements = "@%s//misc/codegen:requirements.txt" % repository_name,
     )
+    bazel_skylib_workspace()
     rules_pkg_dependencies()

misc/codegen/BUILD.bazel

@@ -0,0 +1,14 @@
+load("@codegen_deps//:requirements.bzl", "requirement")
+
+py_binary(
+    name = "codegen",
+    srcs = ["codegen.py"],
+    data = [
+        "//misc/codegen/templates:cpp",
+        "//misc/codegen/templates:trap",
+    ],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//misc/codegen/generators",
+    ],
+)

misc/codegen/README.md

@@ -3,7 +3,7 @@
 This directory contains the code generation suite used by the Swift extractor and the QL library. This suite will use
 the abstract class specification of [`schema.yml`](schema.yml) to generate:
 
-* [the `dbscheme` file](../ql/lib/swift.dbscheme) (see [`dbschemegen.py`](generators/dbschemegen.py))
+* [the `dbscheme` file](../ql/lib/misc.dbscheme) (see [`dbschemegen.py`](generators/dbschemegen.py))
 * [the QL generated code](../ql/lib/codeql/swift/generated) and when
   appropriate [the corresponding stubs](../ql/lib/codeql/swift/elements) (see [`qlgen.py`](generators/qlgen.py))
 * C++ tags and trap entries (see [`trapgen.py`](generators/trapgen.py))
@@ -11,9 +11,9 @@ the abstract class specification of [`schema.yml`](schema.yml) to generate:
 
 ## Usage
 
-By default `bazel run //swift/codegen` will update all checked-in generated files (`dbscheme` and QL sources). You can
+By default `bazel run //misc/codegen` will update all checked-in generated files (`dbscheme` and QL sources). You can
 append `--` followed by other options to tweak the behaviour, which is mainly intended for debugging.
-See `bazel run //swift/codegen -- --help` for a list of all options. In particular `--generate` can be used with a comma
+See `bazel run //misc/codegen -- --help` for a list of all options. In particular `--generate` can be used with a comma
 separated list to select what to generate (choosing among `dbscheme`, `ql`, `trap` and `cpp`).
 
 C++ code is generated during build (see [`swift/extractor/trap/BUILD.bazel`](../extractor/trap/BUILD.bazel)). After a
@@ -38,7 +38,7 @@ specified as the `generate` function in the modules within [the `generators` dir
 Finally, [`codegen.py`](codegen.py) is the driver script gluing everything together and specifying the command line
 options.
 
-Unit tests are in [the `test` directory](test) and can be run via `bazel test //swift/codegen/test`.
+Unit tests are in [the `test` directory](test) and can be run via `bazel test //misc/codegen/test`.
 
 For more details about each specific generation target, please refer to the module docstrings
 in [the `generators` directory](generators).

misc/codegen/codegen.py

@@ -14,8 +14,8 @@ if 'BUILD_WORKSPACE_DIRECTORY' not in os.environ:
     _repo_root = pathlib.Path(__file__).resolve().parents[2]
     sys.path.append(str(_repo_root))
 
-from swift.codegen.lib import render, paths
+from misc.codegen.lib import render, paths
-from swift.codegen.generators import generate
+from misc.codegen.generators import generate
 
 
 def _parse_args() -> argparse.Namespace:

misc/codegen/generators/BUILD.bazel

@@ -0,0 +1,11 @@
+load("@codegen_deps//:requirements.bzl", "requirement")
+
+py_library(
+    name = "generators",
+    srcs = glob(["*.py"]),
+    visibility = ["//misc/codegen:__subpackages__"],
+    deps = [
+        "//misc/codegen/lib",
+        "//misc/codegen/loaders",
+    ],
+)

misc/codegen/generators/cppgen.py

@@ -16,8 +16,8 @@ import typing
 
 import inflection
 
-from swift.codegen.lib import cpp, schema
+from misc.codegen.lib import cpp, schema
-from swift.codegen.loaders import schemaloader
+from misc.codegen.loaders import schemaloader
 
 
 def _get_type(t: str, add_or_none_except: typing.Optional[str] = None) -> str:

misc/codegen/generators/dbschemegen.py

@@ -17,9 +17,9 @@ import typing
 
 import inflection
 
-from swift.codegen.lib import schema
+from misc.codegen.lib import schema
-from swift.codegen.loaders import schemaloader
+from misc.codegen.loaders import schemaloader
-from swift.codegen.lib.dbscheme import *
+from misc.codegen.lib.dbscheme import *
 
 log = logging.getLogger(__name__)
 

misc/codegen/generators/qlgen.py

@@ -29,8 +29,8 @@ import itertools
 
 import inflection
 
-from swift.codegen.lib import schema, ql
+from misc.codegen.lib import schema, ql
-from swift.codegen.loaders import schemaloader
+from misc.codegen.loaders import schemaloader
 
 log = logging.getLogger(__name__)
 

misc/codegen/generators/trapgen.py

@@ -17,8 +17,8 @@ import pathlib
 import inflection
 from toposort import toposort_flatten
 
-from swift.codegen.lib import dbscheme, cpp
+from misc.codegen.lib import dbscheme, cpp
-from swift.codegen.loaders import dbschemeloader
+from misc.codegen.loaders import dbschemeloader
 
 log = logging.getLogger(__name__)
 

misc/codegen/lib/BUILD.bazel

@@ -1,9 +1,9 @@
-load("@swift_codegen_deps//:requirements.bzl", "requirement")
+load("@codegen_deps//:requirements.bzl", "requirement")
 
 py_library(
     name = "lib",
     srcs = glob(["*.py"]),
-    visibility = ["//swift/codegen:__subpackages__"],
+    visibility = ["//misc/codegen:__subpackages__"],
     deps = [
         requirement("pystache"),
         requirement("inflection"),

misc/codegen/lib/paths.py

@@ -4,15 +4,16 @@ import pathlib
 import sys
 import os
 
+_this_file = pathlib.Path(__file__).resolve()
+
 try:
     workspace_dir = pathlib.Path(os.environ['BUILD_WORKSPACE_DIRECTORY']).resolve()  # <- means we are using bazel run
     root_dir = workspace_dir / 'swift'
 except KeyError:
-    _this_file = pathlib.Path(__file__).resolve()
     root_dir = _this_file.parents[2]
     workspace_dir = root_dir.parent
 
-lib_dir = root_dir / 'codegen' / 'lib'
+lib_dir = _this_file.parents[2] / 'codegen' / 'lib'
-templates_dir = root_dir / 'codegen' / 'templates'
+templates_dir = _this_file.parents[2] / 'codegen' / 'templates'
 
 exe_file = pathlib.Path(sys.argv[0]).resolve()

misc/codegen/lib/schemadefs.py

@@ -1,5 +1,5 @@
 from typing import Callable as _Callable
-from swift.codegen.lib import schema as _schema
+from misc.codegen.lib import schema as _schema
 import inspect as _inspect
 from dataclasses import dataclass as _dataclass
 

misc/codegen/loaders/BUILD.bazel

@@ -1,9 +1,9 @@
-load("@swift_codegen_deps//:requirements.bzl", "requirement")
+load("@codegen_deps//:requirements.bzl", "requirement")
 
 py_library(
     name = "loaders",
     srcs = glob(["*.py"]),
-    visibility = ["//swift/codegen:__subpackages__"],
+    visibility = ["//misc/codegen:__subpackages__"],
     deps = [
         requirement("toposort"),
         requirement("inflection"),

misc/codegen/loaders/dbschemeloader.py

@@ -1,6 +1,6 @@
 import pathlib
 import re
-from swift.codegen.lib import dbscheme
+from misc.codegen.lib import dbscheme
 
 
 class _Re:

misc/codegen/loaders/schemaloader.py

@@ -8,7 +8,7 @@ import importlib.util
 from dataclasses import dataclass
 from toposort import toposort_flatten
 
-from swift.codegen.lib import schema, schemadefs
+from misc.codegen.lib import schema, schemadefs
 
 
 @dataclass
@@ -99,7 +99,7 @@ def load(m: types.ModuleType) -> schema.Schema:
     classes = {}
     known = {"int", "string", "boolean"}
     known.update(n for n in m.__dict__ if not n.startswith("__"))
-    import swift.codegen.lib.schemadefs as defs
+    import misc.codegen.lib.schemadefs as defs
     null = None
     for name, data in m.__dict__.items():
         if hasattr(defs, name):

misc/codegen/schemadefs.py

@@ -1,5 +1,5 @@
 from typing import Callable as _Callable
-from swift.codegen.lib import schema as _schema
+from misc.codegen.lib import schema as _schema
 import inspect as _inspect
 from dataclasses import dataclass as _dataclass
 

misc/codegen/templates/BUILD.bazel

@@ -1,4 +1,4 @@
-package(default_visibility = ["//swift:__subpackages__"])
+package(default_visibility = ["//misc/codegen:__subpackages__"])
 
 filegroup(
     name = "trap",

misc/codegen/test/BUILD.bazel

@@ -1,11 +1,11 @@
-load("@swift_codegen_deps//:requirements.bzl", "requirement")
+load("@codegen_deps//:requirements.bzl", "requirement")
 
 py_library(
     name = "utils",
     testonly = True,
     srcs = ["utils.py"],
     deps = [
-        "//swift/codegen/lib",
+        "//misc/codegen/lib",
         requirement("pytest"),
     ],
 )
@@ -17,7 +17,7 @@ py_library(
         srcs = [src],
         deps = [
             ":utils",
-            "//swift/codegen/generators",
+            "//misc/codegen/generators",
         ],
     )
     for src in glob(["test_*.py"])

misc/codegen/test/test_cpp.py

@@ -3,7 +3,7 @@ from copy import deepcopy
 
 import pytest
 
-from swift.codegen.lib import cpp
+from misc.codegen.lib import cpp
 
 
 @pytest.mark.parametrize("keyword", cpp.cpp_keywords)

misc/codegen/test/test_cppgen.py

@@ -1,8 +1,8 @@
 import sys
 
-from swift.codegen.generators import cppgen
+from misc.codegen.generators import cppgen
-from swift.codegen.lib import cpp
+from misc.codegen.lib import cpp
-from swift.codegen.test.utils import *
+from misc.codegen.test.utils import *
 
 output_dir = pathlib.Path("path", "to", "output")
 

misc/codegen/test/test_dbscheme.py

@@ -1,8 +1,8 @@
 import sys
 from copy import deepcopy
 
-from swift.codegen.lib import dbscheme
+from misc.codegen.lib import dbscheme
-from swift.codegen.test.utils import *
+from misc.codegen.test.utils import *
 
 
 def test_dbcolumn_name():

misc/codegen/test/test_dbschemegen.py

@@ -1,9 +1,9 @@
 import collections
 import sys
 
-from swift.codegen.generators import dbschemegen
+from misc.codegen.generators import dbschemegen
-from swift.codegen.lib import dbscheme
+from misc.codegen.lib import dbscheme
-from swift.codegen.test.utils import *
+from misc.codegen.test.utils import *
 
 InputExpectedPair = collections.namedtuple("InputExpectedPair", ("input", "expected"))
 

misc/codegen/test/test_dbschemelaoder.py

@@ -1,9 +1,9 @@
 import sys
 from copy import deepcopy
 
-from swift.codegen.lib import dbscheme
+from misc.codegen.lib import dbscheme
-from swift.codegen.loaders.dbschemeloader import iterload
+from misc.codegen.loaders.dbschemeloader import iterload
-from swift.codegen.test.utils import *
+from misc.codegen.test.utils import *
 
 
 @pytest.fixture

misc/codegen/test/test_ql.py

@@ -1,8 +1,8 @@
 import sys
 from copy import deepcopy
 
-from swift.codegen.lib import ql
+from misc.codegen.lib import ql
-from swift.codegen.test.utils import *
+from misc.codegen.test.utils import *
 
 
 def test_property_has_first_table_param_marked():

misc/codegen/test/test_qlgen.py

@@ -4,9 +4,9 @@ import sys
 
 import pytest
 
-from swift.codegen.generators import qlgen
+from misc.codegen.generators import qlgen
-from swift.codegen.lib import ql
+from misc.codegen.lib import ql
-from swift.codegen.test.utils import *
+from misc.codegen.test.utils import *
 
 
 @pytest.fixture(autouse=True)

misc/codegen/test/test_render.py

@@ -2,7 +2,7 @@ import sys
 
 import pytest
 
-from swift.codegen.test.utils import *
+from misc.codegen.test.utils import *
 
 import hashlib
 

misc/codegen/test/test_schemaloader.py

@@ -2,9 +2,9 @@ import sys
 
 import pytest
 
-from swift.codegen.test.utils import *
+from misc.codegen.test.utils import *
-from swift.codegen.lib import schemadefs as defs
+from misc.codegen.lib import schemadefs as defs
-from swift.codegen.loaders.schemaloader import load
+from misc.codegen.loaders.schemaloader import load
 
 
 def test_empty_schema():

misc/codegen/test/test_trapgen.py

@@ -1,8 +1,8 @@
 import sys
 
-from swift.codegen.generators import trapgen
+from misc.codegen.generators import trapgen
-from swift.codegen.lib import cpp, dbscheme
+from misc.codegen.lib import cpp, dbscheme
-from swift.codegen.test.utils import *
+from misc.codegen.test.utils import *
 
 output_dir = pathlib.Path("path", "to", "output")
 

misc/codegen/test/utils.py

@@ -3,7 +3,7 @@ from unittest import mock
 
 import pytest
 
-from swift.codegen.lib import render, schema, paths
+from misc.codegen.lib import render, schema, paths
 
 schema_dir = pathlib.Path("a", "dir")
 schema_file = schema_dir / "schema.py"
@@ -39,15 +39,15 @@ def opts():
 
 @pytest.fixture(autouse=True)
 def override_paths(tmp_path):
-    with mock.patch("swift.codegen.lib.paths.root_dir", tmp_path), \
+    with mock.patch("misc.codegen.lib.paths.root_dir", tmp_path), \
-            mock.patch("swift.codegen.lib.paths.exe_file", tmp_path / "exe"):
+            mock.patch("misc.codegen.lib.paths.exe_file", tmp_path / "exe"):
         yield
 
 
 @pytest.fixture
 def input(opts, tmp_path):
     opts.schema = tmp_path / schema_file
-    with mock.patch("swift.codegen.loaders.schemaloader.load_file") as load_mock:
+    with mock.patch("misc.codegen.loaders.schemaloader.load_file") as load_mock:
         load_mock.return_value = schema.Schema([])
         yield load_mock.return_value
     assert load_mock.mock_calls == [
@@ -58,7 +58,7 @@ def input(opts, tmp_path):
 @pytest.fixture
 def dbscheme_input(opts, tmp_path):
     opts.dbscheme = tmp_path / dbscheme_file
-    with mock.patch("swift.codegen.loaders.dbschemeloader.iterload") as load_mock:
+    with mock.patch("misc.codegen.loaders.dbschemeloader.iterload") as load_mock:
         load_mock.entities = []
         load_mock.side_effect = lambda _: load_mock.entities
         yield load_mock

swift/README.md

@@ -39,6 +39,8 @@ bazel run //swift/codegen
 to update generated files. This can be shortened to
 `bazel run codegen` if you are in the `swift` directory.
 
+You can also run `../misc/codegen/codegen.py`, as long as you are beneath the `swift` directory.
+
 ## IDE setup
 
 ### CLion and the native bazel plugin

swift/actions/build-and-test/action.yml

@@ -62,7 +62,7 @@ runs:
       if : ${{ github.event_name == 'pull_request' }}
       shell: bash
       run: |
-        bazel test //swift/codegen/test
+        bazel test //misc/codegen/test
     - name: Run qltest tests
       if : ${{ github.event_name == 'pull_request' }}
       shell: bash

swift/codegen.conf

@@ -5,3 +5,5 @@
 --ql-stub-output=ql/lib/codeql/swift/elements
 --ql-test-output=ql/test/extractor-tests/generated
 --generated-registry=ql/.generated.list
+--script-name=codegen/codegen.py
+--trap-library=swift/extractor/trap

swift/codegen/BUILD.bazel

@@ -1,19 +1,15 @@
-load("@swift_codegen_deps//:requirements.bzl", "requirement")
+load("@bazel_skylib//rules:native_binary.bzl", "native_binary")
 
-py_binary(
+native_binary(
     name = "codegen",
-    srcs = ["codegen.py"],
+    out = "codegen",
+    src = "//misc/codegen",
     data = [
         "//swift:schema",
         "//swift:codegen_conf",
-        "//swift/codegen/templates:cpp",
-        "//swift/codegen/templates:trap",
     ],
     args = [
         "--configuration-file=$(location //swift:codegen_conf)",
     ],
     visibility = ["//swift:__subpackages__"],
-    deps = [
-        "//swift/codegen/generators",
-    ],
 )

swift/codegen/generators/BUILD.bazel

@@ -1,11 +0,0 @@
-load("@swift_codegen_deps//:requirements.bzl", "requirement")
-
-py_library(
-    name = "generators",
-    srcs = glob(["*.py"]),
-    visibility = ["//swift/codegen:__subpackages__"],
-    deps = [
-        "//swift/codegen/lib",
-        "//swift/codegen/loaders",
-    ],
-)

swift/extractor/trap/BUILD.bazel

@@ -14,7 +14,7 @@ genrule(
         for ext in ("h", "cpp")
     ],
     cmd = " ".join([
-        "$(location //swift/codegen)",
+        "$(location //misc/codegen)",
         "--generate=dbscheme,trap,cpp",
         "--dbscheme=$(RULEDIR)/generated/swift.dbscheme",
         "--cpp-output=$(RULEDIR)/generated",
@@ -23,7 +23,7 @@ genrule(
         "--schema=$(location //swift:schema)",
         "--script-name=codegen/codegen.py",
     ]),
-    exec_tools = ["//swift/codegen", "//swift:schema"],
+    exec_tools = ["//misc/codegen", "//swift:schema"],
 )
 
 filegroup(

swift/schema.py

@@ -9,7 +9,7 @@ This file should be kept simple:
 For how documentation of generated QL code works, please read schema_documentation.md.
 """
 
-from swift.codegen.lib.schemadefs import *
+from misc.codegen.lib.schemadefs import *
 
 include("prefix.dbscheme")