Bazel: add codeql specific packaging library

This encapsulate arch specific logic, local installation and separation
of zip files into generic and arch-specific parts as required by the
internal build.

misc/bazel/internal/BUILD.bazel

@@ -0,0 +1 @@
+exports_files(["install.py"])

misc/bazel/internal/install.py

@@ -0,0 +1,25 @@
+import argparse
+import pathlib
+import shutil
+import subprocess
+from python.runfiles import runfiles
+
+runfiles = runfiles.Create()
+if not runfiles:
+    raise Exception("Installer should be run with `bazel run`")
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--destdir", type=pathlib.Path, required=True)
+parser.add_argument("--script", required=True)
+parser.add_argument("--build-file", required=True)
+opts = parser.parse_args()
+
+script = runfiles.Rlocation(opts.script)
+build_file = runfiles.Rlocation(opts.build_file)
+destdir = pathlib.Path(build_file).parent / opts.destdir
+
+if destdir.exists():
+    shutil.rmtree(destdir)
+
+destdir.mkdir(parents=True)
+subprocess.run([script, "--destdir", destdir], check=True)

misc/bazel/pkg.bzl

@@ -0,0 +1,264 @@
+"""
+Wrappers and helpers around `rules_pkg` to build codeql packs.
+"""
+
+load("@rules_pkg//pkg:install.bzl", "pkg_install")
+load("@rules_pkg//pkg:mappings.bzl", "pkg_attributes", "pkg_filegroup", "pkg_files", _strip_prefix = "strip_prefix")
+load("@rules_pkg//pkg:pkg.bzl", "pkg_zip")
+load("@rules_python//python:defs.bzl", "py_binary")
+load("//:defs.bzl", "codeql_platform")
+
+def _make_internal(name):
+    def internal(suffix):
+        return "%s-%s" % (name, suffix)
+
+    return internal
+
+def _get_subrule(label, suffix):
+    if ":" in label or "/" not in label:
+        return "%s-%s" % (label, suffix)
+    path, _, pkg = label.rpartition("/")
+    return "%s/%s:%s-%s" % (path, pkg, pkg, suffix)
+
+def codeql_pkg_files(
+        *,
+        name,
+        arch_specific = False,
+        srcs = None,
+        exes = None,
+        renames = None,
+        prefix = None,
+        visibility = None,
+        **kwargs):
+    """
+    Wrapper around `pkg_files`. Added functionality:
+    * `exes` get their file attributes set to be executable. This is important only for POSIX files, there's no need
+      to mark windows executables as such
+    * `arch_specific` auto-adds the codeql platform specific directory (linux64, osx64 or windows64), and will be
+      consumed by a downstream `codeql_pack` to create the arch specific zip.
+    This should be consumed by `codeql_pkg_filegroup` and `codeql_pack` only.
+    """
+    internal = _make_internal(name)
+    main_rule = internal("generic")
+    empty_rule = internal("arch")
+    if arch_specific:
+        main_rule, empty_rule = empty_rule, main_rule
+        prefix = (prefix + "/" if prefix else "") + codeql_platform
+    pkg_files(
+        name = empty_rule,
+        srcs = [],
+        visibility = visibility,
+    )
+    if not srcs and not exes:
+        fail("either srcs or exes should be specified for %s" % name)
+    if srcs and exes:
+        if renames:
+            src_renames = {k: v for k, v in renames.items() if k in srcs}
+            exe_renames = {k: v for k, v in renames.items() if k in exes}
+        else:
+            src_renames = None
+            exe_renames = None
+        pkg_files(
+            name = internal("srcs"),
+            srcs = srcs,
+            renames = src_renames,
+            prefix = prefix,
+            visibility = ["//visibility:private"],
+            **kwargs
+        )
+        pkg_files(
+            name = internal("exes"),
+            srcs = exes,
+            renames = exe_renames,
+            prefix = prefix,
+            visibility = ["//visibility:private"],
+            attributes = pkg_attributes(mode = "0755"),
+            **kwargs
+        )
+        pkg_filegroup(
+            name = main_rule,
+            srcs = [internal("srcs"), internal("exes")],
+            visibility = visibility,
+        )
+    else:
+        pkg_files(
+            name = main_rule,
+            srcs = srcs or exes,
+            attributes = pkg_attributes(mode = "0755") if exes else None,
+            prefix = prefix,
+            visibility = visibility,
+            **kwargs
+        )
+    native.filegroup(
+        name = name,
+        srcs = [main_rule],
+        visibility = visibility,
+    )
+
+def codeql_pkg_wrap(*, name, srcs, arch_specific = False, prefix = None, visibility = None, **kwargs):
+    """
+    Wrap a native `rules_pkg` rule, providing the `arch_specific` functionality and making it consumable by
+    `codeql_pkg_filegroup` and `codeql_pack`.
+    """
+    internal = _make_internal(name)
+    main_rule = internal("generic")
+    empty_rule = internal("arch")
+    if arch_specific:
+        main_rule, empty_rule = empty_rule, main_rule
+        prefix = (prefix + "/" if prefix else "") + codeql_platform
+    pkg_filegroup(
+        name = main_rule,
+        srcs = srcs,
+        prefix = prefix,
+        visibility = visibility,
+        **kwargs
+    )
+    pkg_files(
+        name = empty_rule,
+        srcs = [],
+        visibility = visibility,
+    )
+    native.filegroup(
+        name = name,
+        srcs = [main_rule],
+        visibility = visibility,
+    )
+
+def codeql_pkg_filegroup(
+        *,
+        name,
+        srcs,
+        srcs_select = None,
+        visibility = None,
+        **kwargs):
+    """
+    Combine `codeql_pkg_files` and other `codeql_pkg_filegroup` rules, similar to what `pkg_filegroup` does.
+    `srcs` is not selectable, but `srcs_select` accepts the same dictionary that a select would accept.
+    """
+    internal = _make_internal(name)
+
+    def transform(srcs, suffix):
+        return [_get_subrule(src, suffix) for src in srcs]
+
+    pkg_filegroup(
+        name = internal("generic"),
+        srcs = transform(srcs, "generic") + (select(
+            {k: transform(v, "generic") for k, v in srcs_select.items()},
+        ) if srcs_select else []),
+        visibility = visibility,
+        **kwargs
+    )
+    pkg_filegroup(
+        name = internal("arch"),
+        srcs = transform(srcs, "arch") + (select(
+            {k: transform(v, "arch") for k, v in srcs_select.items()},
+        ) if srcs_select else []),
+        visibility = visibility,
+        **kwargs
+    )
+    native.filegroup(
+        name = name,
+        srcs = [internal("generic"), internal("arch")],
+        visibility = visibility,
+    )
+
+def codeql_pack(*, name, srcs, zip_prefix = None, zip_filename = "extractor", visibility = visibility, install_dest = "extractor-pack", **kwargs):
+    """
+    Define a codeql pack. This accepts the same arguments as `codeql_pkg_filegroup`, and additionally:
+    * defines a `<name>-generic-zip` target creating a `<zip_filename>-generic.zip` archive with the generic bits,
+      prefixed with `zip_prefix` (`name` by default)
+    * defines a `<name>-arch-zip` target creating a `<zip_filename>-<codeql_platform>.zip` archive with the
+      arch-specific bits, prefixed with `zip_prefix` (`name` by default)
+    * defines a runnable `<name>-installer` target that will install the pack in `install_dest`, relative to where the
+      rule is used. The install destination can be overridden appending `-- --destdir=...` to the `bazel run`
+      invocation. This installation does not use the `zip_prefix`.
+    """
+    internal = _make_internal(name)
+    zip_prefix = zip_prefix or name
+    zip_filename = zip_filename or name
+    codeql_pkg_filegroup(
+        name = name,
+        srcs = srcs,
+        visibility = visibility,
+        **kwargs
+    )
+    codeql_pkg_filegroup(
+        name = internal("zip-contents"),
+        srcs = [name],
+        prefix = zip_prefix,
+        visibility = ["//visibility:private"],
+    )
+    pkg_zip(
+        name = internal("generic-zip"),
+        srcs = [internal("zip-contents-generic")],
+        package_file_name = zip_filename + "-generic.zip",
+        visibility = visibility,
+    )
+    pkg_zip(
+        name = internal("arch-zip"),
+        srcs = [internal("zip-contents-arch")],
+        package_file_name = zip_filename + "-" + codeql_platform + ".zip",
+        visibility = visibility,
+    )
+    pkg_install(
+        name = internal("script"),
+        srcs = [internal("generic"), internal("arch")],
+        visibility = ["//visibility:private"],
+    )
+    native.filegroup(
+        # used to locate current source directory
+        name = internal("build-file"),
+        srcs = ["BUILD.bazel"],
+        visibility = ["//visibility:private"],
+    )
+    py_binary(
+        name = internal("installer"),
+        srcs = ["//misc/bazel/internal:install.py"],
+        main = "//misc/bazel/internal:install.py",
+        data = [internal("build-file"), internal("script")],
+        deps = ["@rules_python//python/runfiles"],
+        args = [
+            "--build-file=$(rlocationpath %s)" % internal("build-file"),
+            "--script=$(rlocationpath %s)" % internal("script"),
+            "--destdir",
+            install_dest,
+        ],
+        visibility = visibility,
+    )
+
+strip_prefix = _strip_prefix
+
+def _runfiles_group_impl(ctx):
+    files = []
+    for src in ctx.attr.srcs:
+        rf = src[DefaultInfo].default_runfiles
+        if rf != None:
+            files.append(rf.files)
+    return [
+        DefaultInfo(
+            files = depset(transitive = files),
+        ),
+    ]
+
+_runfiles_group = rule(
+    implementation = _runfiles_group_impl,
+    attrs = {
+        "srcs": attr.label_list(),
+    },
+)
+
+def codeql_pkg_runfiles(*, name, exes, **kwargs):
+    """
+    Create a `codeql_pkg_files` with all runfiles from files in `exes`, flattened together.
+    """
+    internal = _make_internal(name)
+    _runfiles_group(
+        name = internal("runfiles"),
+        srcs = exes,
+        visibility = ["//visibility:private"],
+    )
+    codeql_pkg_files(
+        name = name,
+        exes = [internal("runfiles")],
+        **kwargs
+    )

misc/bazel/pkg_runfiles.bzl

@@ -1,33 +0,0 @@
-load("@rules_pkg//pkg:mappings.bzl", "pkg_attributes", "pkg_files")
-
-def _runfiles_group_impl(ctx):
-    files = []
-    for src in ctx.attr.srcs:
-        rf = src[DefaultInfo].default_runfiles
-        if rf != None:
-            files.append(rf.files)
-    return [
-        DefaultInfo(
-            files = depset(transitive = files),
-        ),
-    ]
-
-_runfiles_group = rule(
-    implementation = _runfiles_group_impl,
-    attrs = {
-        "srcs": attr.label_list(),
-    },
-)
-
-def pkg_runfiles(*, name, srcs, **kwargs):
-    internal_name = "_%s_runfiles" % name
-    _runfiles_group(
-        name = internal_name,
-        srcs = srcs,
-    )
-    kwargs.setdefault("attributes", pkg_attributes(mode = "0755"))
-    pkg_files(
-        name = name,
-        srcs = [internal_name],
-        **kwargs
-    )

swift/BUILD.bazel

@@ -1,8 +1,11 @@
-load("@rules_pkg//pkg:install.bzl", "pkg_install")
-load("@rules_pkg//pkg:mappings.bzl", "pkg_filegroup", "pkg_files")
-load("//:defs.bzl", "codeql_platform")
-load("//misc/bazel:pkg_runfiles.bzl", "pkg_runfiles")
-load("//misc/bazel/cmake:cmake.bzl", "generate_cmake")
+load(
+    "//misc/bazel:pkg.bzl",
+    "codeql_pack",
+    "codeql_pkg_filegroup",
+    "codeql_pkg_files",
+    "codeql_pkg_runfiles",
+    "codeql_pkg_wrap",
+)
 
 filegroup(
     name = "schema",
@@ -22,7 +25,7 @@ filegroup(
     visibility = ["//swift:__subpackages__"],
 )
 
-pkg_files(
+codeql_pkg_files(
     name = "dbscheme_files",
     srcs = [
         "ql/lib/swift.dbscheme.stats",
@@ -30,100 +33,78 @@ pkg_files(
     ],
 )
 
-pkg_files(
+codeql_pkg_files(
     name = "manifest",
     srcs = ["codeql-extractor.yml"],
 )
 
-pkg_filegroup(
-    name = "extractor-pack-generic",
-    srcs = [
-        ":manifest",
-        "//swift/tools",
-    ] + select({
-        "@platforms//os:windows": [],
-        "//conditions:default": [
-            ":dbscheme_files",
-            "//swift/downgrades",
-        ],
-    }),
-    visibility = ["//visibility:public"],
-)
-
-pkg_filegroup(
+codeql_pkg_filegroup(
     name = "extractor",
     srcs = ["//swift/extractor:pkg"],
-    prefix = "tools/" + codeql_platform,
+    prefix = "tools",
 )
 
-pkg_runfiles(
-    name = "swift-autobuilder",
-    srcs = ["//swift/swift-autobuilder"],
-    prefix = "tools/" + codeql_platform,
+codeql_pkg_files(
+    name = "autobuilder-incompatible-os",
+    arch_specific = True,
+    exes = ["//swift/tools/diagnostics:autobuilder-incompatible-os"],
+    prefix = "tools",
 )
 
-pkg_runfiles(
-    name = "diagnostics",
-    srcs = ["//swift/tools/diagnostics:autobuilder-incompatible-os"],
-    prefix = "tools/" + codeql_platform,
+codeql_pkg_runfiles(
+    name = "autobuilder",
+    arch_specific = True,
+    exes = ["//swift/swift-autobuilder"],
+    prefix = "tools",
 )
 
-pkg_filegroup(
-    name = "resource-dir-arch",
+codeql_pkg_wrap(
+    name = "resource-dir",
     srcs = ["//swift/third_party/swift-llvm-support:swift-resource-dir"],
-    prefix = "resource-dir/" + codeql_platform,
-    visibility = ["//visibility:public"],
+    arch_specific = True,
+    prefix = "resource-dir",
 )
 
-pkg_filegroup(
-    name = "extractor-pack-arch",
-    srcs = select({
-        "@platforms//os:windows": [],
-        "//conditions:default": [
-            ":extractor",
-            ":resource-dir-arch",
-        ],
-    }) + select({
-        "@platforms//os:macos": [
-            ":swift-autobuilder",
-        ],
-        "//conditions:default": [
-            ":diagnostics",
-        ],
-    }),
-    visibility = ["//visibility:public"],
-)
-
-pkg_filegroup(
-    name = "extractor-pack",
+codeql_pack(
+    name = "swift",
     srcs = [
-        ":extractor-pack-arch",
-        ":extractor-pack-generic",
+        ":dbscheme_files",
+        ":manifest",
+        "//swift/downgrades",
+        "//swift/tools",
     ],
+    srcs_select = {
+        "@platforms//os:macos": [
+            ":extractor",
+            ":resource-dir",
+            ":autobuilder",
+        ],
+        "@platforms//os:linux": [
+            ":extractor",
+            ":resource-dir",
+            ":autobuilder-incompatible-os",
+        ],
+        "@platforms//os:windows": [
+            ":autobuilder-incompatible-os",
+        ],
+    },
     visibility = ["//visibility:public"],
 )
 
-pkg_install(
-    name = "_create_extractor_pack",
-    srcs = ["//swift:extractor-pack"],
-)
-
-py_binary(
+alias(
     name = "create-extractor-pack",
-    srcs = ["create_extractor_pack.py"],
-    main = "create_extractor_pack.py",
-    deps = [":_create_extractor_pack"],
+    actual = ":swift-installer",
 )
 
-# TODO this is unneeded here but still used in the internal repo. Remove once it's not
-generate_cmake(
-    name = "cmake",
-    targets = [
-        "//swift/extractor:extractor.real",
-        "//swift/logging/tests/assertion-diagnostics:assert-false",
-    ] + select({
-        "@platforms//os:linux": ["//swift/tools/diagnostics:autobuilder-incompatible-os"],
-        "@platforms//os:macos": ["//swift/swift-autobuilder"],
-    }),
+# TODO: aliases for internal repo backward compatibility
+alias(
+    name = "extractor-pack-generic",
+    actual = "swift-generic",
+    visibility = ["//visibility:public"],
+)
+
+alias(
+    name = "extractor-pack-arch",
+    actual = "swift-arch",
     visibility = ["//visibility:public"],
 )

swift/downgrades/BUILD.bazel

@@ -1,6 +1,6 @@
-load("@rules_pkg//pkg:mappings.bzl", "pkg_files", "strip_prefix")
+load("//misc/bazel:pkg.bzl", "codeql_pkg_files", "strip_prefix")
 
-pkg_files(
+codeql_pkg_files(
     name = "downgrades",
     srcs = glob(
         ["**"],

swift/extractor/BUILD.bazel

@@ -1,4 +1,4 @@
-load("//misc/bazel:pkg_runfiles.bzl", "pkg_runfiles")
+load("//misc/bazel:pkg.bzl", "codeql_pkg_runfiles")
 load("//swift:rules.bzl", "swift_cc_binary")
 
 swift_cc_binary(
@@ -29,9 +29,10 @@ sh_binary(
     data = [":extractor.real"],
 )
 
-pkg_runfiles(
+codeql_pkg_runfiles(
     name = "pkg",
-    srcs = [":extractor"],
+    arch_specific = True,
     excludes = ["extractor.sh"],  # script gets copied as "extractor", no need for the original .sh file
+    exes = [":extractor"],
     visibility = ["//swift:__pkg__"],
 )

swift/tools/BUILD.bazel

@@ -1,4 +1,4 @@
-load("@rules_pkg//pkg:mappings.bzl", "pkg_attributes", "pkg_filegroup", "pkg_files")
+load("//misc/bazel:pkg.bzl", "codeql_pkg_files")
 
 sh_binary(
     name = "qltest",
@@ -16,29 +16,17 @@ sh_binary(
     srcs = ["identify-environment.sh"],
 )
 
-pkg_files(
-    name = "scripts",
+codeql_pkg_files(
+    name = "tools",
     srcs = [
         "autobuild.cmd",
+        "tracing-config.lua",
+    ],
+    exes = [
         ":autobuild",
         ":identify-environment",
         ":qltest",
     ],
-    attributes = pkg_attributes(mode = "0755"),
     prefix = "tools",
-)
-
-pkg_files(
-    name = "tracing-config",
-    srcs = ["tracing-config.lua"],
-    prefix = "tools",
-)
-
-pkg_filegroup(
-    name = "tools",
-    srcs = [
-        ":scripts",
-        ":tracing-config",
-    ],
     visibility = ["//swift:__pkg__"],
 )