Swift: rework workflows

* A unique workflow file has been created merging all `swift-*.yml` workflows * Change filtering at job level was added using [dorny/paths-filter][1] * only one build of the extractor is made, and then shared via cache (not as an artifact because of [this longstading issue][2]) * integration tests are now run on on macOS * qltests are not run any more on macOS to cut on feedback time * autobuilder tests were moved to the macOS build step to avoid loading bazel twice [1]: https://github.com/dorny/paths-filter#examples [2]: https://github.com/actions/upload-artifact/issues/38
2025-12-17 01:03:14 +01:00 · 2022-10-27 09:58:07 +02:00
parent 21600c612d
commit 003866621f
9 changed files with 205 additions and 175 deletions
--- a/.github/workflows/swift-autobuilder.yml
+++ b/.github/workflows/swift-autobuilder.yml
@@ -1,27 +0,0 @@
-name: "Swift: Build and test Xcode autobuilder"
-
-on:
-  pull_request:
-    paths:
-      - "swift/xcode-autobuilder/**"
-      - "misc/bazel/**"
-      - "*.bazel*"
-      - .github/workflows/swift-autobuilder.yml
-    branches:
-      - main
-
-jobs:
-  autobuilder:
-    runs-on: macos-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: bazelbuild/setup-bazelisk@v2
-      - uses: actions/setup-python@v4
-        with:
-          python-version-file: 'swift/.python-version'
-      - name: Build the Xcode autobuilder
-        run: |
-          bazel build //swift/xcode-autobuilder
-      - name: Test the Xcode autobuilder
-        run: |
-          bazel test //swift/xcode-autobuilder/tests
--- a/.github/workflows/swift-codegen.yml
+++ b/.github/workflows/swift-codegen.yml
@@ -1,44 +0,0 @@
-name: "Swift: Check code generation"
-
-on:
-  pull_request:
-    paths:
-      - "swift/**"
-      - "misc/bazel/**"
-      - "*.bazel*"
-      - .github/workflows/swift-codegen.yml
-      - .github/actions/fetch-codeql/action.yml
-    branches:
-      - main
-defaults:
-  run:
-    working-directory: swift
-
-jobs:
-  codegen:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./.github/actions/fetch-codeql
-      - uses: bazelbuild/setup-bazelisk@v2
-      - uses: actions/setup-python@v4
-        with:
-          python-version-file: 'swift/.python-version'
-      - uses: pre-commit/action@v3.0.0
-        name: Check that python code is properly formatted
-        with:
-          extra_args: autopep8 --all-files
-      - name: Run unit tests
-        run: |
-          bazel test //swift/codegen/test --test_output=errors
-      - uses: pre-commit/action@v3.0.0
-        name: Check that QL generated code was checked in
-        with:
-          extra_args: swift-codegen --all-files
-      - name: Generate C++ files
-        run: |
-          bazel run //swift/codegen:codegen -- --generate=trap,cpp --cpp-output=$PWD/swift-generated-cpp-files
-      - uses: actions/upload-artifact@v3
-        with:
-          name: swift-generated-cpp-files
-          path: swift-generated-cpp-files/**
--- a/.github/workflows/swift-integration-tests.yml
+++ b/.github/workflows/swift-integration-tests.yml
@@ -1,47 +0,0 @@
-name: "Swift: Run Integration Tests"
-
-on:
-  pull_request:
-    paths:
-      - "swift/**"
-      - "misc/bazel/**"
-      - "*.bazel*"
-      - .github/workflows/swift-integration-tests.yml
-      - .github/actions/fetch-codeql/action.yml
-      - codeql-workspace.yml
-    branches:
-      - main
-defaults:
-  run:
-    working-directory: swift
-
-jobs:
-  integration-tests:
-    runs-on: ${{ matrix.os }}
-    strategy:
-      fail-fast: false
-      matrix:
-        os:
-          - ubuntu-20.04
-#          - macos-latest  TODO
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./.github/actions/fetch-codeql
-      - uses: bazelbuild/setup-bazelisk@v2
-      - uses: actions/setup-python@v4
-        with:
-          python-version-file: 'swift/.python-version'
-      - name: Build Swift extractor
-        run: |
-          bazel run //swift:create-extractor-pack
-      - name: Get Swift version
-        id: get_swift_version
-        run: |
-          VERSION=$(bazel run //swift/extractor -- --version | sed -ne 's/.*version \(\S*\).*/\1/p')
-          echo "::set-output name=version::$VERSION"
-      - uses: swift-actions/setup-swift@v1
-        with:
-          swift-version: "${{steps.get_swift_version.outputs.version}}"
-      - name: Run integration tests
-        run: |
-          python integration-tests/runner.py
--- a/.github/workflows/swift-qltest.yml
+++ b/.github/workflows/swift-qltest.yml
@@ -1,57 +0,0 @@
-name: "Swift: Run QL Tests"
-
-on:
-  pull_request:
-    paths:
-      - "swift/**"
-      - "misc/bazel/**"
-      - "*.bazel*"
-      - .github/workflows/swift-qltest.yml
-      - .github/actions/fetch-codeql/action.yml
-      - codeql-workspace.yml
-    branches:
-      - main
-defaults:
-  run:
-    working-directory: swift
-
-jobs:
-  qlformat:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./.github/actions/fetch-codeql
-      - name: Check QL formatting
-        run: find ql "(" -name "*.ql" -or -name "*.qll" ")" -print0 | xargs -0 codeql query format --check-only
-  qltest-test:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: bazelbuild/setup-bazelisk@v2
-      - uses: actions/setup-python@v4
-        with:
-          python-version-file: 'swift/.python-version'
-      - name: Test qltest.sh
-        run: |
-          bazel test //swift/tools/test/qltest
-  qltest:
-    runs-on: ${{ matrix.os }}
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [ ubuntu-20.04, macos-latest ]
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./.github/actions/fetch-codeql
-      - uses: bazelbuild/setup-bazelisk@v2
-      - uses: actions/setup-python@v4
-        with:
-          python-version-file: 'swift/.python-version'
-      - name: Build Swift extractor
-        run: |
-          bazel run //swift:create-extractor-pack
-      - name: Run QL tests
-        run: |
-          codeql test run --threads=0 --ram 5000 --search-path "${{ github.workspace }}/swift/extractor-pack" --check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition ql/test
-        env:
-          GITHUB_TOKEN: ${{ github.token }}
--- a/.github/workflows/swift.yml
+++ b/.github/workflows/swift.yml
@@ -0,0 +1,137 @@
+name: "Swift"
+
+on:
+  pull_request:
+    paths:
+      - "swift/**"
+      - "misc/bazel/**"
+      - "*.bazel*"
+      - .github/workflows/swift.yml
+      - .github/actions/fetch-codeql/action.yml
+      - codeql-workspace.yml
+      - .pre-commit-config.yaml
+      - "!**/*.md"
+      - "!**/*.qhelp"
+    branches:
+      - main
+defaults:
+  run:
+    working-directory: swift
+
+jobs:
+  changes:
+    runs-on: ubuntu-latest
+    outputs:
+      codegen: ${{ steps.filter.outputs.codegen }}
+      ql: ${{ steps.filter.outputs.ql }}
+    steps:
+      - uses: dorny/paths-filter@v2
+        id: filter
+        with:
+          filters: |
+            codegen:
+              - 'github/workflows/swift.yml'
+              - "misc/bazel/**"
+              - "*.bazel*"
+              - 'swift/actions/env-setup/**'
+              - '.pre-commit-config.yaml'
+              - 'swift/codegen/**'
+              - 'swift/schema.py'
+              - 'swift/**/*.dbscheme'
+              - 'swift/ql/lib/codeql/swift/elements.qll'
+              - 'swift/ql/lib/codeql/swift/elements/**'
+              - 'swift/ql/lib/codeql/swift/generated/**'
+              - 'swift/ql/test/extractor-tests/generated/**'
+            ql:
+              - 'github/workflows/swift.yml'
+              - 'swift/**/*.ql'
+              - 'swift/**/*.qll'
+  # not using a matrix as you cannot depend on a specific job in a matrix, and we want to start linux checks
+  # without waiting for the macOS build
+  build-macos:
+    runs-on: macos-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./swift/actions/create-extractor-pack
+      # loading bazel targets is very long on the basic macOS runner
+      # therefore it's better to test the X
+      - name: Test the Xcode autobuilder
+        run: |
+          bazel test //swift/xcode-autobuilder/tests
+  build-linux:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./swift/actions/create-extractor-pack
+  qltests:
+    needs: build-linux
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./swift/actions/env-setup
+      - uses: ./swift/actions/share-extractor-pack
+      - name: Test qltest.sh
+        run: |
+          bazel test //swift/tools/test/qltest
+      - uses: ./.github/actions/fetch-codeql
+      - name: Run QL tests
+        run: |
+          codeql test run \
+            --threads=0 \
+            --ram 5000 \
+            --search-path "${{ github.workspace }}/swift/extractor-pack" \
+            --check-databases \
+            --check-unused-labels \
+            --check-repeated-labels \
+            --check-redefined-labels \
+            --check-use-before-definition \
+            ql/test
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+  integration-tests-linux:
+    needs: build-linux
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./swift/actions/run-integration-tests
+  integration-tests-macos:
+    needs: build-macos
+    runs-on: macos-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./swift/actions/run-integration-tests
+  codegen:
+    runs-on: ubuntu-latest
+    needs: changes
+    if: ${{ needs.changes.outputs.codegen == 'true' }}
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./swift/actions/env-setup
+      - uses: pre-commit/action@v3.0.0
+        name: Check that python code is properly formatted
+        with:
+          extra_args: autopep8 --all-files
+      - name: Run unit tests
+        run: |
+          bazel test //swift/codegen/test --test_output=errors
+      - uses: ./.github/actions/fetch-codeql
+      - uses: pre-commit/action@v3.0.0
+        name: Check that QL generated code was checked in
+        with:
+          extra_args: swift-codegen --all-files
+      - name: Generate C++ files
+        run: |
+          bazel run //swift/codegen:codegen -- --generate=trap,cpp --cpp-output=$PWD/generated-cpp-files
+      - uses: actions/upload-artifact@v3
+        with:
+          name: swift-generated-cpp-files
+          path: swift/generated-cpp-files/**
+  qlformat:
+    runs-on: ubuntu-latest
+    needs: changes
+    if: ${{ needs.changes.outputs.ql == 'true' }}
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/fetch-codeql
+      - name: Check QL formatting
+        run: find ql "(" -name "*.ql" -or -name "*.qll" ")" -print0 | xargs -0 codeql query format --check-only
--- a/swift/actions/create-extractor-pack/action.yml
+++ b/swift/actions/create-extractor-pack/action.yml
@@ -0,0 +1,11 @@
+name: Build Swift CodeQL pack
+description: Builds the Swift CodeQL pack
+runs:
+  using: composite
+  steps:
+    - uses: ./swift/actions/env-setup
+    - uses: ./swift/actions/share-extractor-pack
+    - name: Build Swift extractor
+      shell: bash
+      run: |
+        bazel run //swift:create-extractor-pack
--- a/swift/actions/env-setup/action.yml
+++ b/swift/actions/env-setup/action.yml
@@ -0,0 +1,23 @@
+name: Setup Swift Package Environment
+description: Sets up the environment in which to build the Swift package and run its tests
+runs:
+  using: composite
+  steps:
+    - uses: bazelbuild/setup-bazelisk@v2
+    - uses: actions/setup-python@v4
+      with:
+        python-version-file: 'swift/.python-version'
+    - name: Mount bazel cache
+      uses: actions/cache@v3
+      with:
+        path: "~/.cache/bazel-repository-cache"
+        key: bazel-cache-${{ runner.os }}-${{ runner.arch }}
+    - name: Mount bazel disk cache
+      uses: actions/cache@v3
+      with:
+        path: "~/.cache/bazel-disk-cache"
+        key: bazel-disk-cache-${{ runner.os }}-${{ runner.arch }}
+    - name: Set up bazel disk cache
+      shell: bash
+      run: |
+        echo build --repository_cache=~/.cache/bazel-repository-cache --disk_cache=~/.cache/bazel-disk-cache > ~/.bazelrc
--- a/swift/actions/run-integration-tests/action.yml
+++ b/swift/actions/run-integration-tests/action.yml
@@ -0,0 +1,23 @@
+name: Build Swift CodeQL pack
+description: Builds the Swift CodeQL pack
+runs:
+  using: composite
+  steps:
+    - uses: ./swift/actions/share-extractor-pack
+    - name: Get Swift version
+      id: get_swift_version
+      shell: bash
+      run: |
+        VERSION=$(swift/extractor-pack/tools/*/extractor --version | awk '/version/ { print $3 }')
+        echo "version=$VERSION" | tee -a $GITHUB_OUTPUT
+    - uses: actions/setup-python@v4
+      with:
+        python-version-file: 'swift/.python-version'
+    - uses: swift-actions/setup-swift@v1
+      with:
+        swift-version: "${{steps.get_swift_version.outputs.version}}"
+    - uses: ./.github/actions/fetch-codeql
+    - name: Run integration tests
+      shell: bash
+      run: |
+        python swift/integration-tests/runner.py
--- a/swift/actions/share-extractor-pack/action.yml
+++ b/swift/actions/share-extractor-pack/action.yml
@@ -0,0 +1,11 @@
+name: Build Swift CodeQL pack
+description: Builds the Swift CodeQL pack
+runs:
+  using: composite
+  steps:
+    # not using artifacts because of annoying https://github.com/actions/upload-artifact/issues/38
+    - name: Mount cache for sharing extractor pack
+      uses: actions/cache@v3
+      with:
+        path: swift/extractor-pack
+        key: extractor-pack-${{ github.run_id }}-${{ github.run_number }}-${{ runner.os }}