Swift: try out new builds

.gitattributes

@@ -86,4 +86,5 @@
 /misc/ripunzip/ripunzip-* filter=lfs diff=lfs merge=lfs -text
 
 # swift prebuilt resources
-/swift/third_party/resource-dir/*.zip filter=lfs diff=lfs merge=lfs -text
+/swift/third_party/resources/*.zip filter=lfs diff=lfs merge=lfs -text
+/swift/third_party/resources/*.tar.zst filter=lfs diff=lfs merge=lfs -text

.github/workflows/swift.yml

@@ -109,3 +109,9 @@ jobs:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/fetch-codeql
       - uses: ./swift/actions/database-upgrade-scripts
+  check-no-override:
+    if : ${{ github.event_name == 'pull_request' }}
+    runs-on: ubuntu-latest
+    steps:
+      - shell: bash
+        run: bazel test //swift/... --test_tag_filters=override --test_output=errors

MODULE.bazel

@@ -28,6 +28,7 @@ bazel_dep(name = "gazelle", version = "0.38.0")
 bazel_dep(name = "rules_dotnet", version = "0.15.1")
 bazel_dep(name = "googletest", version = "1.14.0.bcr.1")
 bazel_dep(name = "rules_rust", version = "0.50.0")
+bazel_dep(name = "zstd", version = "1.5.5.bcr.1")
 
 bazel_dep(name = "buildifier_prebuilt", version = "6.4.0", dev_dependency = True)
 
@@ -91,10 +92,10 @@ use_repo(
     swift_deps,
     "binlog",
     "picosha2",
-    "swift_prebuilt_darwin_x86_64",
-    "swift_prebuilt_linux",
-    "swift_toolchain_linux",
-    "swift_toolchain_macos",
+    "swift-prebuilt-linux",
+    "swift-prebuilt-macos",
+    "swift-resource-dir-linux",
+    "swift-resource-dir-macos",
 )
 
 node = use_extension("@rules_nodejs//nodejs:extensions.bzl", "node")
@@ -186,16 +187,6 @@ lfs_files(
     executable = True,
 )
 
-lfs_files(
-    name = "swift-resource-dir-linux",
-    srcs = ["//swift/third_party/resource-dir:resource-dir-linux.zip"],
-)
-
-lfs_files(
-    name = "swift-resource-dir-macos",
-    srcs = ["//swift/third_party/resource-dir:resource-dir-macos.zip"],
-)
-
 register_toolchains(
     "@nodejs_toolchains//:all",
 )

misc/bazel/lfs.bzl

@@ -77,6 +77,12 @@ def _download_lfs(repository_ctx):
         )
     repository_ctx.file("BUILD.bazel", build)
 
+    # this is for drop-in compatibility with `http_file`
+    repository_ctx.file(
+        "file/BUILD.bazel",
+        'alias(name = "file", actual = "//:%s", visibility = ["//visibility:public"])\n' % name,
+    )
+
 lfs_archive = repository_rule(
     doc = "Export the contents from an on-demand LFS archive. The corresponding path should be added to be ignored " +
           "in `.lfsconfig`.",

swift/BUILD.bazel

@@ -81,7 +81,7 @@ codeql_pack(
     zips = select({
         "@platforms//os:windows": {},
         "//conditions:default": {
-            "//swift/third_party/resource-dir": "resource-dir/{CODEQL_PLATFORM}",
+            "//swift/third_party/resources:dir": "resource-dir/{CODEQL_PLATFORM}",
         },
     }),
 )

swift/actions/build-and-test/action.yml

@@ -58,7 +58,7 @@ runs:
       if: ${{ github.event_name == 'pull_request' }}
       shell: bash
       run: |
-        bazel test //swift/...
+        bazel test //swift/... --test_tag_filters=-override --test_output=errors
     - name: Evict bazel cache
       if: ${{ github.event_name != 'pull_request' }}
       shell: bash

swift/extractor/BUILD.bazel

@@ -18,7 +18,7 @@ swift_cc_binary(
         "//swift/extractor/invocation",
         "//swift/extractor/remapping",
         "//swift/extractor/translators",
-        "//swift/third_party/swift-llvm-support",
+        "//swift/third_party/resources:prebuilt",
         "@absl//absl/strings",
     ],
 )

swift/extractor/infra/BUILD.bazel

@@ -10,7 +10,7 @@ swift_cc_library(
         "//swift/extractor/infra/file",
         "//swift/extractor/trap",
         "//swift/logging",
-        "//swift/third_party/swift-llvm-support",
+        "//swift/third_party/resources:prebuilt",
         "@picosha2",
     ],
 )

swift/extractor/main.cpp

@@ -20,6 +20,7 @@
 #include "swift/extractor/trap/TrapDomain.h"
 #include "swift/extractor/infra/file/Path.h"
 #include "swift/logging/SwiftAssert.h"
+#include "swift/Threading/Errors.h"
 
 using namespace std::string_literals;
 using namespace codeql::main_logger;

swift/extractor/mangler/BUILD.bazel

@@ -8,6 +8,6 @@ swift_cc_library(
     deps = [
         "//swift/extractor/infra",
         "//swift/extractor/trap",
-        "//swift/third_party/swift-llvm-support",
+        "//swift/third_party/resources:prebuilt",
     ],
 )

swift/extractor/remapping/BUILD.bazel

@@ -8,7 +8,7 @@ swift_cc_library(
     deps = [
         "//swift/extractor/config",
         "//swift/extractor/infra/file",
-        "//swift/third_party/swift-llvm-support",
+        "//swift/third_party/resources:prebuilt",
         "@picosha2",
     ],
 )

swift/extractor/translators/BUILD.bazel

@@ -8,6 +8,6 @@ swift_cc_library(
     deps = [
         "//swift/extractor/infra",
         "//swift/extractor/mangler",
-        "//swift/third_party/swift-llvm-support",
+        "//swift/third_party/resources:prebuilt",
     ],
 )

swift/third_party/BUILD.bazel

@@ -0,0 +1,3 @@
+load(":load.bzl", "test_no_override")
+
+test_no_override()

swift/third_party/BUILD.swift-llvm-support.bazel

@@ -1,12 +1,9 @@
 cc_library(
-    name = "swift-llvm-support",
+    name = "swift-llvm-support-static",
     srcs = glob(
         [
             "*.a",
-            "*.so",
-            "*.dylib",
         ],
-        allow_empty = True,  # Either *.so or *.dylib will be empty
     ),
     hdrs = glob([
         "include/**/*",
@@ -16,6 +13,20 @@ cc_library(
         "include",
         "stdlib/public/SwiftShims",
     ],
+    deps = [
+        "@zstd",
+    ],
+)
+
+cc_library(
+    name = "swift-llvm-support",
+    srcs = glob(
+        [
+            "*.so",
+            "*.dylib",
+        ],
+        allow_empty = True,  # Either *.so or *.dylib will be empty
+    ),
     linkopts = [
         "-lm",
         "-lz",
@@ -32,4 +43,5 @@ cc_library(
         "//conditions:default": [],
     }),
     visibility = ["//visibility:public"],
+    deps = [":swift-llvm-support-static"],
 )

swift/third_party/load.bzl

@@ -1,101 +1,55 @@
-load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+load("@bazel_skylib//rules:write_file.bzl", "write_file")
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive", "http_file")
 load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe")
+load("//misc/bazel:lfs.bzl", "lfs_archive", "lfs_files")
 
-_swift_prebuilt_version = "swift-6.0.1-RELEASE.330"
-_swift_sha_map = {
-    "Linux-X64": "3da9b257b08da3bed023656c3bea2e1d0e6504b1592f593a077023c59e5339fc",
-    "macOS-X64": "66641b3b285e593342b88d48defa6668b15a85603acfe5aba5b62b9ed9123465",
+# these are used to test new artifacts. They must not be merged to main as different from None
+_override_resource_dir = {
+    "macOS": "ad533e614c3565db17186fa93684bd404d1bd66120b563957a44afc997a82b5e",
+    "Linux": "d6f1abbe9c0662ec2418b9a8c0136b1d8399601f556631a7b0910115cef3a38a",
+}
+_override_prebuilt = {
+    "macOS": "8f3c775aa7a62e97046f4dcfbc5b51c317712250396c7a07f7d0f4bd666a59d4",
+    "Linux": "5658fe92fe60b01b897757495d455c9fe435037a0973cb5b642e04be00a77ed3",
 }
 
-_swift_arch_map = {
-    "Linux-X64": "linux",
-    "macOS-X64": "darwin_x86_64",
-}
+_staging_url = "https://github.com/dsp-testing/codeql-swift-artifacts/releases/download/staging-{file}/{file}"
 
-_swift_version = _swift_prebuilt_version.rpartition(".")[0]
-
-_toolchain_info = {
-    "linux": struct(
-        platform = "ubuntu2004",
-        suffix = "ubuntu20.04",
-        extension = "tar.gz",
-        sha = "935d0b68757d9b1aceb6410fe0b126a28a07e362553ebba0c4bcd1c9a55d0bc5",
-    ),
-    "macos": struct(
-        platform = "xcode",
-        suffix = "osx",
-        extension = "pkg",
-        sha = "ef9bb6b38711324e1b1c89de44a27d9519d0711924c57f4df541734b04aaf6cc",
-    ),
-}
-
-def _get_toolchain_url(info):
-    return "https://download.swift.org/%s/%s/%s/%s-%s.%s" % (
-        _swift_version.lower(),
-        info.platform,
-        _swift_version,
-        _swift_version,
-        info.suffix,
-        info.extension,
+def _load_resource_dir(plat):
+    name = "swift-resource-dir-%s" % plat.lower()
+    file = "resource-dir-%s.zip" % plat
+    override = _override_resource_dir[plat]
+    if override:
+        http_file(
+            name = name,
+            url = _staging_url.format(file = file),
+            sha256 = override,
+            downloaded_file_path = file,
+        )
+    else:
+        lfs_files(
+            name = name,
+            srcs = ["//swift/third_party/resources:%s" % file],
         )
 
-def _toolchains():
-    rules = {
-        "tar.gz": http_archive,
-        "pkg": _pkg_archive,
-    }
-    for arch, info in _toolchain_info.items():
-        rule = rules[info.extension]
-        rule(
-            name = "swift_toolchain_%s" % arch,
-            url = _get_toolchain_url(info),
-            sha256 = info.sha,
-            build_file = _build % "swift-toolchain-%s" % arch,
-            strip_prefix = "%s-%s" % (_swift_version, info.suffix),
+def _load_prebuilt(plat):
+    name = "swift-prebuilt-%s" % plat.lower()
+    file = "swift-prebuilt-%s.tar.zst" % plat
+    build = _build % "swift-llvm-support"
+    override = _override_prebuilt[plat]
+    if override:
+        http_archive(
+            name = name,
+            url = _staging_url.format(file = file),
+            sha256 = override,
+            build_file = build,
         )
-
-def _run(repository_ctx, message, cmd, working_directory = "."):
-    repository_ctx.report_progress(message)
-    res = repository_ctx.execute(
-        ["bash", "-c", cmd],
-        working_directory = working_directory,
+    else:
+        lfs_archive(
+            name = name,
+            src = "//swift/third_party/resources:%s" % file,
+            build_file = build,
         )
-    if res.return_code != 0:
-        fail(message)
-
-def _pkg_archive_impl(repository_ctx):
-    archive = "file.pkg"
-    url = repository_ctx.attr.url
-    dir = "%s-package.pkg" % repository_ctx.attr.strip_prefix
-    repository_ctx.report_progress("downloading %s" % url)
-    res = repository_ctx.download(
-        url,
-        output = archive,
-        sha256 = repository_ctx.attr.sha256,
-    )
-    if not repository_ctx.attr.sha256:
-        print("Rule '%s' indicated that a canonical reproducible form " % repository_ctx.name +
-              "can be obtained by modifying arguments sha256 = \"%s\"" % res.sha256)
-    _run(repository_ctx, "extracting %s" % dir, "xar -xf %s" % archive)
-    repository_ctx.delete(archive)
-    _run(
-        repository_ctx,
-        "extracting Payload from %s" % dir,
-        "cat %s/Payload | gunzip -dc | cpio -i" % dir,
-    )
-    repository_ctx.delete(dir)
-    repository_ctx.symlink(repository_ctx.attr.build_file, "BUILD")
-    repository_ctx.file("WORKSPACE")
-
-_pkg_archive = repository_rule(
-    implementation = _pkg_archive_impl,
-    attrs = {
-        "url": attr.string(mandatory = True),
-        "sha256": attr.string(),
-        "strip_prefix": attr.string(),
-        "build_file": attr.label(mandatory = True),
-    },
-)
 
 def _github_archive(*, name, repository, commit, build_file = None, sha256 = None):
     github_name = repository[repository.index("/") + 1:]
@@ -111,20 +65,9 @@ def _github_archive(*, name, repository, commit, build_file = None, sha256 = Non
 _build = "//swift/third_party:BUILD.%s.bazel"
 
 def load_dependencies(module_ctx):
-    for repo_arch, arch in _swift_arch_map.items():
-        sha256 = _swift_sha_map[repo_arch]
-
-        http_archive(
-            name = "swift_prebuilt_%s" % arch,
-            url = "https://github.com/dsp-testing/codeql-swift-artifacts/releases/download/%s/swift-prebuilt-%s.zip" % (
-                _swift_prebuilt_version,
-                repo_arch,
-            ),
-            build_file = _build % "swift-llvm-support",
-            sha256 = sha256,
-        )
-
-    _toolchains()
+    for plat in ("macOS", "Linux"):
+        _load_prebuilt(plat)
+        _load_resource_dir(plat)
 
     _github_archive(
         name = "picosha2",
@@ -148,3 +91,28 @@ def load_dependencies(module_ctx):
     )
 
 swift_deps = module_extension(load_dependencies)
+
+def test_no_override():
+    test_body = ["#!/bin/bash", "", "RET=0"]
+    for name, definition in (
+        ("_override_prebuilt", _override_prebuilt),
+        ("_override_resource_dir", _override_resource_dir),
+    ):
+        for plat in ("macOS", "Linux"):
+            if definition[plat]:
+                test_body += [
+                    'echo %s[\\"%s\\"] overridden in swift/third/party/load.bzl' % (name, plat),
+                    "RET=1",
+                ]
+    test_body += ["", "exit $RET"]
+    write_file(
+        name = "test-no-override-gen",
+        out = "test-no-override.sh",
+        content = test_body,
+        is_executable = True,
+    )
+    native.sh_test(
+        name = "test-no-override",
+        srcs = [":test-no-override-gen"],
+        tags = ["override"],
+    )

swift/third_party/resource-dir/BUILD.bazel

@@ -1,27 +0,0 @@
-alias(
-    name = "resource-dir",
-    actual = select({"@platforms//os:" + os: "@swift-resource-dir-" + os for os in ("linux", "macos")}),
-    target_compatible_with = select({
-        "@platforms//os:windows": ["@platforms//:incompatible"],
-        "//conditions:default": [],
-    }),
-    visibility = ["//visibility:public"],
-)
-
-[
-    sh_binary(
-        name = "update-" + os,
-        srcs = ["update.sh"],
-        args = [
-            "$(rlocationpath @swift_toolchain_%s)" % os,
-            "$(rlocationpath resource-dir-%s.zip)" % os,
-        ],
-        data = [
-            "resource-dir-%s.zip" % os,
-            "@swift_toolchain_" + os,
-        ],
-        target_compatible_with = ["@platforms//os:" + os],
-        deps = ["//misc/bazel:sh_runfiles"],
-    )
-    for os in ("linux", "macos")
-]

swift/third_party/resource-dir/README.md

@@ -1,2 +0,0 @@
-These LFS files are redistributed parts of the [Swift toolchains](https://www.swift.org/download/).
-A [copy](./LICENSE.txt) of the [swift](https://github.com/apple/swift) license is included.

swift/third_party/resources/BUILD.bazel

@@ -0,0 +1,21 @@
+_oses = ("linux", "macos")
+
+alias(
+    name = "dir",
+    actual = select({"@platforms//os:" + os: "@swift-resource-dir-%s//file" % os for os in _oses}),
+    target_compatible_with = select({
+        "@platforms//os:windows": ["@platforms//:incompatible"],
+        "//conditions:default": [],
+    }),
+    visibility = ["//visibility:public"],
+)
+
+alias(
+    name = "prebuilt",
+    actual = select({"@platforms//os:" + os: "@swift-prebuilt-%s//:swift-llvm-support" % os for os in _oses}),
+    target_compatible_with = select({
+        "@platforms//os:windows": ["@platforms//:incompatible"],
+        "//conditions:default": [],
+    }),
+    visibility = ["//visibility:public"],
+)

swift/third_party/resources/README.md

@@ -0,0 +1,7 @@
+The `resource-dir-*.zip` LFS files are redistributed parts of the [Swift toolchains](https://www.swift.org/download/).
+
+The `swift-prebuilt-*.tar.zst` LFS files are precompiled binaries of the [swift code][swift].
+
+A [copy](./LICENSE.txt) of the [swift][] license is included.
+
+[swift]: https://github.com/apple/swift

swift/third_party/swift-llvm-support/BUILD.bazel

@@ -1,9 +0,0 @@
-package(default_visibility = ["//swift:__subpackages__"])
-
-alias(
-    name = "swift-llvm-support",
-    actual = select({
-        "@bazel_tools//src/conditions:linux": "@swift_prebuilt_linux//:swift-llvm-support",
-        "@bazel_tools//src/conditions:darwin": "@swift_prebuilt_darwin_x86_64//:swift-llvm-support",
-    }),
-)