diff --git a/.gitattributes b/.gitattributes index 215fe0b81e1..df5bed028be 100644 --- a/.gitattributes +++ b/.gitattributes @@ -86,4 +86,5 @@ /misc/ripunzip/ripunzip-* filter=lfs diff=lfs merge=lfs -text # swift prebuilt resources -/swift/third_party/resource-dir/*.zip filter=lfs diff=lfs merge=lfs -text +/swift/third_party/resources/*.zip filter=lfs diff=lfs merge=lfs -text +/swift/third_party/resources/*.tar.zst filter=lfs diff=lfs merge=lfs -text diff --git a/.github/workflows/swift.yml b/.github/workflows/swift.yml index ad7e8f52aa3..6056dc4363f 100644 --- a/.github/workflows/swift.yml +++ b/.github/workflows/swift.yml @@ -109,3 +109,9 @@ jobs: - uses: actions/checkout@v4 - uses: ./.github/actions/fetch-codeql - uses: ./swift/actions/database-upgrade-scripts + check-no-override: + if : ${{ github.event_name == 'pull_request' }} + runs-on: ubuntu-latest + steps: + - shell: bash + run: bazel test //swift/... --test_tag_filters=override --test_output=errors diff --git a/MODULE.bazel b/MODULE.bazel index 4a2219d43a5..b4f209dae7c 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -28,6 +28,7 @@ bazel_dep(name = "gazelle", version = "0.38.0") bazel_dep(name = "rules_dotnet", version = "0.15.1") bazel_dep(name = "googletest", version = "1.14.0.bcr.1") bazel_dep(name = "rules_rust", version = "0.50.0") +bazel_dep(name = "zstd", version = "1.5.5.bcr.1") bazel_dep(name = "buildifier_prebuilt", version = "6.4.0", dev_dependency = True) @@ -91,10 +92,10 @@ use_repo( swift_deps, "binlog", "picosha2", - "swift_prebuilt_darwin_x86_64", - "swift_prebuilt_linux", - "swift_toolchain_linux", - "swift_toolchain_macos", + "swift-prebuilt-linux", + "swift-prebuilt-macos", + "swift-resource-dir-linux", + "swift-resource-dir-macos", ) node = use_extension("@rules_nodejs//nodejs:extensions.bzl", "node") @@ -186,16 +187,6 @@ lfs_files( executable = True, ) -lfs_files( - name = "swift-resource-dir-linux", - srcs = ["//swift/third_party/resource-dir:resource-dir-linux.zip"], -) - -lfs_files( - name = "swift-resource-dir-macos", - srcs = ["//swift/third_party/resource-dir:resource-dir-macos.zip"], -) - register_toolchains( "@nodejs_toolchains//:all", ) diff --git a/misc/bazel/lfs.bzl b/misc/bazel/lfs.bzl index a068d76b2ea..5dcba3894a3 100644 --- a/misc/bazel/lfs.bzl +++ b/misc/bazel/lfs.bzl @@ -77,6 +77,12 @@ def _download_lfs(repository_ctx): ) repository_ctx.file("BUILD.bazel", build) + # this is for drop-in compatibility with `http_file` + repository_ctx.file( + "file/BUILD.bazel", + 'alias(name = "file", actual = "//:%s", visibility = ["//visibility:public"])\n' % name, + ) + lfs_archive = repository_rule( doc = "Export the contents from an on-demand LFS archive. The corresponding path should be added to be ignored " + "in `.lfsconfig`.", diff --git a/swift/BUILD.bazel b/swift/BUILD.bazel index 444730950be..52509a0963c 100644 --- a/swift/BUILD.bazel +++ b/swift/BUILD.bazel @@ -81,7 +81,7 @@ codeql_pack( zips = select({ "@platforms//os:windows": {}, "//conditions:default": { - "//swift/third_party/resource-dir": "resource-dir/{CODEQL_PLATFORM}", + "//swift/third_party/resources:dir": "resource-dir/{CODEQL_PLATFORM}", }, }), ) diff --git a/swift/actions/build-and-test/action.yml b/swift/actions/build-and-test/action.yml index 2522f545c05..9cbae47bacc 100644 --- a/swift/actions/build-and-test/action.yml +++ b/swift/actions/build-and-test/action.yml @@ -58,7 +58,7 @@ runs: if: ${{ github.event_name == 'pull_request' }} shell: bash run: | - bazel test //swift/... + bazel test //swift/... --test_tag_filters=-override --test_output=errors - name: Evict bazel cache if: ${{ github.event_name != 'pull_request' }} shell: bash diff --git a/swift/extractor/BUILD.bazel b/swift/extractor/BUILD.bazel index 8290aec4121..962392a6c57 100644 --- a/swift/extractor/BUILD.bazel +++ b/swift/extractor/BUILD.bazel @@ -18,7 +18,7 @@ swift_cc_binary( "//swift/extractor/invocation", "//swift/extractor/remapping", "//swift/extractor/translators", - "//swift/third_party/swift-llvm-support", + "//swift/third_party/resources:prebuilt", "@absl//absl/strings", ], ) diff --git a/swift/extractor/infra/BUILD.bazel b/swift/extractor/infra/BUILD.bazel index 6a624844c76..7f6af092ef4 100644 --- a/swift/extractor/infra/BUILD.bazel +++ b/swift/extractor/infra/BUILD.bazel @@ -10,7 +10,7 @@ swift_cc_library( "//swift/extractor/infra/file", "//swift/extractor/trap", "//swift/logging", - "//swift/third_party/swift-llvm-support", + "//swift/third_party/resources:prebuilt", "@picosha2", ], ) diff --git a/swift/extractor/main.cpp b/swift/extractor/main.cpp index ad2939bb5e2..045d7fdb968 100644 --- a/swift/extractor/main.cpp +++ b/swift/extractor/main.cpp @@ -20,6 +20,7 @@ #include "swift/extractor/trap/TrapDomain.h" #include "swift/extractor/infra/file/Path.h" #include "swift/logging/SwiftAssert.h" +#include "swift/Threading/Errors.h" using namespace std::string_literals; using namespace codeql::main_logger; diff --git a/swift/extractor/mangler/BUILD.bazel b/swift/extractor/mangler/BUILD.bazel index 71c9cbf900e..658e750bdcc 100644 --- a/swift/extractor/mangler/BUILD.bazel +++ b/swift/extractor/mangler/BUILD.bazel @@ -8,6 +8,6 @@ swift_cc_library( deps = [ "//swift/extractor/infra", "//swift/extractor/trap", - "//swift/third_party/swift-llvm-support", + "//swift/third_party/resources:prebuilt", ], ) diff --git a/swift/extractor/remapping/BUILD.bazel b/swift/extractor/remapping/BUILD.bazel index da5eee55dc1..44a7d7445bb 100644 --- a/swift/extractor/remapping/BUILD.bazel +++ b/swift/extractor/remapping/BUILD.bazel @@ -8,7 +8,7 @@ swift_cc_library( deps = [ "//swift/extractor/config", "//swift/extractor/infra/file", - "//swift/third_party/swift-llvm-support", + "//swift/third_party/resources:prebuilt", "@picosha2", ], ) diff --git a/swift/extractor/translators/BUILD.bazel b/swift/extractor/translators/BUILD.bazel index 0bfc59db970..7b3356f6960 100644 --- a/swift/extractor/translators/BUILD.bazel +++ b/swift/extractor/translators/BUILD.bazel @@ -8,6 +8,6 @@ swift_cc_library( deps = [ "//swift/extractor/infra", "//swift/extractor/mangler", - "//swift/third_party/swift-llvm-support", + "//swift/third_party/resources:prebuilt", ], ) diff --git a/swift/third_party/BUILD.bazel b/swift/third_party/BUILD.bazel index e69de29bb2d..b9982c37e8d 100644 --- a/swift/third_party/BUILD.bazel +++ b/swift/third_party/BUILD.bazel @@ -0,0 +1,3 @@ +load(":load.bzl", "test_no_override") + +test_no_override() diff --git a/swift/third_party/BUILD.swift-llvm-support.bazel b/swift/third_party/BUILD.swift-llvm-support.bazel index af98184b673..231500b35ab 100644 --- a/swift/third_party/BUILD.swift-llvm-support.bazel +++ b/swift/third_party/BUILD.swift-llvm-support.bazel @@ -1,12 +1,9 @@ cc_library( - name = "swift-llvm-support", + name = "swift-llvm-support-static", srcs = glob( [ "*.a", - "*.so", - "*.dylib", ], - allow_empty = True, # Either *.so or *.dylib will be empty ), hdrs = glob([ "include/**/*", @@ -16,6 +13,20 @@ cc_library( "include", "stdlib/public/SwiftShims", ], + deps = [ + "@zstd", + ], +) + +cc_library( + name = "swift-llvm-support", + srcs = glob( + [ + "*.so", + "*.dylib", + ], + allow_empty = True, # Either *.so or *.dylib will be empty + ), linkopts = [ "-lm", "-lz", @@ -32,4 +43,5 @@ cc_library( "//conditions:default": [], }), visibility = ["//visibility:public"], + deps = [":swift-llvm-support-static"], ) diff --git a/swift/third_party/load.bzl b/swift/third_party/load.bzl index a893b59d2dc..75eb585b6cc 100644 --- a/swift/third_party/load.bzl +++ b/swift/third_party/load.bzl @@ -1,101 +1,55 @@ -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("@bazel_skylib//rules:write_file.bzl", "write_file") +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive", "http_file") load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") +load("//misc/bazel:lfs.bzl", "lfs_archive", "lfs_files") -_swift_prebuilt_version = "swift-6.0.1-RELEASE.330" -_swift_sha_map = { - "Linux-X64": "3da9b257b08da3bed023656c3bea2e1d0e6504b1592f593a077023c59e5339fc", - "macOS-X64": "66641b3b285e593342b88d48defa6668b15a85603acfe5aba5b62b9ed9123465", +# these are used to test new artifacts. They must not be merged to main as different from None +_override_resource_dir = { + "macOS": "ad533e614c3565db17186fa93684bd404d1bd66120b563957a44afc997a82b5e", + "Linux": "d6f1abbe9c0662ec2418b9a8c0136b1d8399601f556631a7b0910115cef3a38a", +} +_override_prebuilt = { + "macOS": "8f3c775aa7a62e97046f4dcfbc5b51c317712250396c7a07f7d0f4bd666a59d4", + "Linux": "5658fe92fe60b01b897757495d455c9fe435037a0973cb5b642e04be00a77ed3", } -_swift_arch_map = { - "Linux-X64": "linux", - "macOS-X64": "darwin_x86_64", -} +_staging_url = "https://github.com/dsp-testing/codeql-swift-artifacts/releases/download/staging-{file}/{file}" -_swift_version = _swift_prebuilt_version.rpartition(".")[0] - -_toolchain_info = { - "linux": struct( - platform = "ubuntu2004", - suffix = "ubuntu20.04", - extension = "tar.gz", - sha = "935d0b68757d9b1aceb6410fe0b126a28a07e362553ebba0c4bcd1c9a55d0bc5", - ), - "macos": struct( - platform = "xcode", - suffix = "osx", - extension = "pkg", - sha = "ef9bb6b38711324e1b1c89de44a27d9519d0711924c57f4df541734b04aaf6cc", - ), -} - -def _get_toolchain_url(info): - return "https://download.swift.org/%s/%s/%s/%s-%s.%s" % ( - _swift_version.lower(), - info.platform, - _swift_version, - _swift_version, - info.suffix, - info.extension, - ) - -def _toolchains(): - rules = { - "tar.gz": http_archive, - "pkg": _pkg_archive, - } - for arch, info in _toolchain_info.items(): - rule = rules[info.extension] - rule( - name = "swift_toolchain_%s" % arch, - url = _get_toolchain_url(info), - sha256 = info.sha, - build_file = _build % "swift-toolchain-%s" % arch, - strip_prefix = "%s-%s" % (_swift_version, info.suffix), +def _load_resource_dir(plat): + name = "swift-resource-dir-%s" % plat.lower() + file = "resource-dir-%s.zip" % plat + override = _override_resource_dir[plat] + if override: + http_file( + name = name, + url = _staging_url.format(file = file), + sha256 = override, + downloaded_file_path = file, + ) + else: + lfs_files( + name = name, + srcs = ["//swift/third_party/resources:%s" % file], ) -def _run(repository_ctx, message, cmd, working_directory = "."): - repository_ctx.report_progress(message) - res = repository_ctx.execute( - ["bash", "-c", cmd], - working_directory = working_directory, - ) - if res.return_code != 0: - fail(message) - -def _pkg_archive_impl(repository_ctx): - archive = "file.pkg" - url = repository_ctx.attr.url - dir = "%s-package.pkg" % repository_ctx.attr.strip_prefix - repository_ctx.report_progress("downloading %s" % url) - res = repository_ctx.download( - url, - output = archive, - sha256 = repository_ctx.attr.sha256, - ) - if not repository_ctx.attr.sha256: - print("Rule '%s' indicated that a canonical reproducible form " % repository_ctx.name + - "can be obtained by modifying arguments sha256 = \"%s\"" % res.sha256) - _run(repository_ctx, "extracting %s" % dir, "xar -xf %s" % archive) - repository_ctx.delete(archive) - _run( - repository_ctx, - "extracting Payload from %s" % dir, - "cat %s/Payload | gunzip -dc | cpio -i" % dir, - ) - repository_ctx.delete(dir) - repository_ctx.symlink(repository_ctx.attr.build_file, "BUILD") - repository_ctx.file("WORKSPACE") - -_pkg_archive = repository_rule( - implementation = _pkg_archive_impl, - attrs = { - "url": attr.string(mandatory = True), - "sha256": attr.string(), - "strip_prefix": attr.string(), - "build_file": attr.label(mandatory = True), - }, -) +def _load_prebuilt(plat): + name = "swift-prebuilt-%s" % plat.lower() + file = "swift-prebuilt-%s.tar.zst" % plat + build = _build % "swift-llvm-support" + override = _override_prebuilt[plat] + if override: + http_archive( + name = name, + url = _staging_url.format(file = file), + sha256 = override, + build_file = build, + ) + else: + lfs_archive( + name = name, + src = "//swift/third_party/resources:%s" % file, + build_file = build, + ) def _github_archive(*, name, repository, commit, build_file = None, sha256 = None): github_name = repository[repository.index("/") + 1:] @@ -111,20 +65,9 @@ def _github_archive(*, name, repository, commit, build_file = None, sha256 = Non _build = "//swift/third_party:BUILD.%s.bazel" def load_dependencies(module_ctx): - for repo_arch, arch in _swift_arch_map.items(): - sha256 = _swift_sha_map[repo_arch] - - http_archive( - name = "swift_prebuilt_%s" % arch, - url = "https://github.com/dsp-testing/codeql-swift-artifacts/releases/download/%s/swift-prebuilt-%s.zip" % ( - _swift_prebuilt_version, - repo_arch, - ), - build_file = _build % "swift-llvm-support", - sha256 = sha256, - ) - - _toolchains() + for plat in ("macOS", "Linux"): + _load_prebuilt(plat) + _load_resource_dir(plat) _github_archive( name = "picosha2", @@ -148,3 +91,28 @@ def load_dependencies(module_ctx): ) swift_deps = module_extension(load_dependencies) + +def test_no_override(): + test_body = ["#!/bin/bash", "", "RET=0"] + for name, definition in ( + ("_override_prebuilt", _override_prebuilt), + ("_override_resource_dir", _override_resource_dir), + ): + for plat in ("macOS", "Linux"): + if definition[plat]: + test_body += [ + 'echo %s[\\"%s\\"] overridden in swift/third/party/load.bzl' % (name, plat), + "RET=1", + ] + test_body += ["", "exit $RET"] + write_file( + name = "test-no-override-gen", + out = "test-no-override.sh", + content = test_body, + is_executable = True, + ) + native.sh_test( + name = "test-no-override", + srcs = [":test-no-override-gen"], + tags = ["override"], + ) diff --git a/swift/third_party/resource-dir/BUILD.bazel b/swift/third_party/resource-dir/BUILD.bazel deleted file mode 100644 index 9cea2efd029..00000000000 --- a/swift/third_party/resource-dir/BUILD.bazel +++ /dev/null @@ -1,27 +0,0 @@ -alias( - name = "resource-dir", - actual = select({"@platforms//os:" + os: "@swift-resource-dir-" + os for os in ("linux", "macos")}), - target_compatible_with = select({ - "@platforms//os:windows": ["@platforms//:incompatible"], - "//conditions:default": [], - }), - visibility = ["//visibility:public"], -) - -[ - sh_binary( - name = "update-" + os, - srcs = ["update.sh"], - args = [ - "$(rlocationpath @swift_toolchain_%s)" % os, - "$(rlocationpath resource-dir-%s.zip)" % os, - ], - data = [ - "resource-dir-%s.zip" % os, - "@swift_toolchain_" + os, - ], - target_compatible_with = ["@platforms//os:" + os], - deps = ["//misc/bazel:sh_runfiles"], - ) - for os in ("linux", "macos") -] diff --git a/swift/third_party/resource-dir/README.md b/swift/third_party/resource-dir/README.md deleted file mode 100644 index 38873b4a54f..00000000000 --- a/swift/third_party/resource-dir/README.md +++ /dev/null @@ -1,2 +0,0 @@ -These LFS files are redistributed parts of the [Swift toolchains](https://www.swift.org/download/). -A [copy](./LICENSE.txt) of the [swift](https://github.com/apple/swift) license is included. diff --git a/swift/third_party/resources/BUILD.bazel b/swift/third_party/resources/BUILD.bazel new file mode 100644 index 00000000000..3bf263911cd --- /dev/null +++ b/swift/third_party/resources/BUILD.bazel @@ -0,0 +1,21 @@ +_oses = ("linux", "macos") + +alias( + name = "dir", + actual = select({"@platforms//os:" + os: "@swift-resource-dir-%s//file" % os for os in _oses}), + target_compatible_with = select({ + "@platforms//os:windows": ["@platforms//:incompatible"], + "//conditions:default": [], + }), + visibility = ["//visibility:public"], +) + +alias( + name = "prebuilt", + actual = select({"@platforms//os:" + os: "@swift-prebuilt-%s//:swift-llvm-support" % os for os in _oses}), + target_compatible_with = select({ + "@platforms//os:windows": ["@platforms//:incompatible"], + "//conditions:default": [], + }), + visibility = ["//visibility:public"], +) diff --git a/swift/third_party/resource-dir/LICENSE.txt b/swift/third_party/resources/LICENSE.txt similarity index 100% rename from swift/third_party/resource-dir/LICENSE.txt rename to swift/third_party/resources/LICENSE.txt diff --git a/swift/third_party/resources/README.md b/swift/third_party/resources/README.md new file mode 100644 index 00000000000..99b48964e19 --- /dev/null +++ b/swift/third_party/resources/README.md @@ -0,0 +1,7 @@ +The `resource-dir-*.zip` LFS files are redistributed parts of the [Swift toolchains](https://www.swift.org/download/). + +The `swift-prebuilt-*.tar.zst` LFS files are precompiled binaries of the [swift code][swift]. + +A [copy](./LICENSE.txt) of the [swift][] license is included. + +[swift]: https://github.com/apple/swift diff --git a/swift/third_party/resource-dir/resource-dir-linux.zip b/swift/third_party/resources/resource-dir-linux.zip similarity index 100% rename from swift/third_party/resource-dir/resource-dir-linux.zip rename to swift/third_party/resources/resource-dir-linux.zip diff --git a/swift/third_party/resource-dir/resource-dir-macos.zip b/swift/third_party/resources/resource-dir-macos.zip similarity index 100% rename from swift/third_party/resource-dir/resource-dir-macos.zip rename to swift/third_party/resources/resource-dir-macos.zip diff --git a/swift/third_party/resource-dir/update.sh b/swift/third_party/resources/update.sh similarity index 100% rename from swift/third_party/resource-dir/update.sh rename to swift/third_party/resources/update.sh diff --git a/swift/third_party/swift-llvm-support/BUILD.bazel b/swift/third_party/swift-llvm-support/BUILD.bazel deleted file mode 100644 index 4bc1fffba94..00000000000 --- a/swift/third_party/swift-llvm-support/BUILD.bazel +++ /dev/null @@ -1,9 +0,0 @@ -package(default_visibility = ["//swift:__subpackages__"]) - -alias( - name = "swift-llvm-support", - actual = select({ - "@bazel_tools//src/conditions:linux": "@swift_prebuilt_linux//:swift-llvm-support", - "@bazel_tools//src/conditions:darwin": "@swift_prebuilt_darwin_x86_64//:swift-llvm-support", - }), -)