Merge pull request #131 from github/esbena/workflow-improvements

Misc. workflow improvements

.github/workflows/bleeding-codeql-analysis.yml

@@ -12,102 +12,17 @@ on:
 
 jobs:
 
-  build_query_pack:
-    runs-on: ubuntu-latest-xl
-    steps:
-      - uses: actions/checkout@v2
-      - name: Find codeql
-        id: find-codeql
-        uses: github/codeql-action/init@esbena/ql
-        with:
-          languages: javascript # does not matter
-      - name: Build query pack
-        run: |
-          cd ql/src
-          "${CODEQL}" pack create
-          cd .codeql/pack/codeql/ql-all/0.0.0
-          zip "${PACKZIP}" -r .
-        env:
-          CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
-          PACKZIP: ${{ runner.temp }}/query-pack.zip
-      - name: Upload query pack
-        uses: actions/upload-artifact@v2
-        with:
-          name: query-pack
-          path: ${{ runner.temp }}/query-pack.zip
-
-  # XXX this is mostly an inlined copy of the 'build' job in build.yml
-  build_extractor_pack:
-    strategy:
-      matrix:
-        os: [ubuntu-latest]
-
-    runs-on: ${{ matrix.os }}
-
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/cache@v2
-        with:
-          path: |
-            ~/.cargo/registry
-            ~/.cargo/git
-            target
-          key: ${{ runner.os }}-rust-cargo-${{ hashFiles('**/Cargo.lock') }}
-      - name: Check formatting
-        run: cargo fmt --all -- --check
-      - name: Build
-        run: cargo build --verbose
-      - name: Run tests
-        run: cargo test --verbose
-      - name: Release build
-        run: cargo build --release
-      - name: Generate dbscheme
-        run: target/release/ql-generator --dbscheme ql/src/ql.dbscheme --library ql/src/codeql_ql/ast/internal/TreeSitter.qll
-      - uses: actions/upload-artifact@v2
-        with:
-          name: ql.dbscheme
-          path: ql/src/ql.dbscheme
-      - uses: actions/upload-artifact@v2
-        with:
-          name: TreeSitter.qll
-          path: ql/src/codeql_ql/ast/internal/TreeSitter.qll
-      - uses: actions/upload-artifact@v2
-        with:
-          name: extractor-${{ matrix.os }}
-          path: |
-            target/release/ql-extractor
-          retention-days: 1
-  # XXX this is mostly an inlined copy of the 'package' job in build.yml
-      - run: |
-          mkdir -p ${PACK_DIR}
-        env:
-          PACK_DIR: ${{ runner.temp }}/pack
-
-      - name: Pack
-        working-directory: ${{ runner.temp }}/pack
-        run: |
-          mkdir -p ql
-          cp -r "${CHECKOUT}/codeql-extractor.yml" "${CHECKOUT}/tools" "${CHECKOUT}/ql/src/ql.dbscheme" "${CHECKOUT}/ql/src/ql.dbscheme.stats" ql/
-          mkdir -p ql/tools/linux64
-          cp "${CHECKOUT}/target/release/ql-extractor" ql/tools/linux64/extractor
-          chmod +x ql/tools/linux64/extractor
-          zip -rq codeql-ql.zip ql
-        env:
-          CHECKOUT: ${{ github.workspace }}
-
-      - uses: actions/upload-artifact@v2
-        with:
-          name: extractor-pack
-          path: ${{ runner.temp }}/pack/codeql-ql.zip
-          retention-days: 1
+  build:
+    uses: github/codeql-ql/.github/workflows/build.yml@esbena/workflow-improvements
+    with:
+      os: '[ "ubuntu-latest" ]'
 
   analyze:
     name: Analyze
     needs:
-      - build_query_pack
-      - build_extractor_pack
+      - build
 
-    runs-on: ubuntu-latest-xl
+    runs-on: ubuntu-latest
 
     permissions:
       actions: read
@@ -115,43 +30,25 @@ jobs:
       security-events: write
 
     steps:
-    - name: Download query pack
+    - name: Download pack
       uses: actions/download-artifact@v2
       with:
-        name: query-pack
-        path: ${{ runner.temp }}/query-pack-artifact
+        name: codeql-ql-pack
+        path: ${{ runner.temp }}/codeql-ql-pack-artifact
 
-    - name: Download extractor pack
-      uses: actions/download-artifact@v2
-      with:
-        name: extractor-pack
-        path: ${{ runner.temp }}/extractor-pack-artifact
-
-    - name: Prepare packs
-      id: prepare-packs
+    - name: Prepare pack
       run: |
-        set -x
-        mkdir -p "${COMPLETE_PACK}" "${PACKS_TMP}"
-        cd "${PACKS_TMP}"
-        unzip "${QUERY_PACK_ARTIFACT}/*.zip" -d query-pack-artifact-unzipped
-        cp -r query-pack-artifact-unzipped/. "${COMPLETE_PACK}"
-        unzip "${EXTRACTOR_PACK_ARTIFACT}/*.zip" -d extractor-pack-artifact-unzipped
-        cp -r extractor-pack-artifact-unzipped/ql/. "${COMPLETE_PACK}"
-        cd "${COMPLETE_PACK}"
-        zip "${COMPLETE_PACK_ZIP}" -r .
+        unzip "${PACK_ARTIFACT}/*.zip" -d "${PACK}"
       env:
-        PACKS_TMP: ${{ runner.temp }}/pack-artifacts.tmp
-        QUERY_PACK_ARTIFACT: ${{ runner.temp }}/query-pack-artifact
-        EXTRACTOR_PACK_ARTIFACT: ${{ runner.temp }}/extractor-pack-artifact
-        COMPLETE_PACK: ${{ runner.temp }}/pack
-        COMPLETE_PACK_ZIP: ${{ runner.temp }}/pack.zip
+        PACK_ARTIFACT: ${{ runner.temp }}/codeql-ql-pack-artifact
+        PACK: ${{ runner.temp }}/pack
 
     - name: Hack codeql-action options
       run: |
-        JSON=$(jq -nc --arg pack "${COMPLETE_PACK}" '.resolve.queries=["--search-path", $pack] | .resolve.extractor=["--search-path", $pack] | .database.init=["--search-path", $pack]')
+        JSON=$(jq -nc --arg pack "${PACK}" '.resolve.queries=["--search-path", $pack] | .resolve.extractor=["--search-path", $pack] | .database.init=["--search-path", $pack]')
         echo "CODEQL_ACTION_EXTRA_OPTIONS=${JSON}" >> ${GITHUB_ENV}
       env:
-        COMPLETE_PACK: ${{ runner.temp }}/pack
+        PACK: ${{ runner.temp }}/pack
 
     - name: Checkout repository
       uses: actions/checkout@v2
@@ -172,9 +69,3 @@ jobs:
         path: ${{ runner.temp }}/db
         retention-days: 1
 
-    - name: Upload complete pack
-      uses: actions/upload-artifact@v2
-      with:
-        name: complete-pack
-        path: ${{ runner.temp }}/pack.zip
-        retention-days: 1

.github/workflows/build.yml

@@ -1,21 +1,50 @@
-name: Rust
+name: Build codeql-ql-pack
 
 on:
   push:
     branches: [main]
   pull_request:
     branches: [main]
+  workflow_call:
+    inputs:
+      os:
+        description: A JSON array string of (fixed) operating systems to build for, e.g. '["ubuntu-latest", "macos-latest", "windows-latest"]'
+        required: false
+        type: string
 
 env:
   CARGO_TERM_COLOR: always
 
 jobs:
-  build:
+  queries:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Find codeql
+        id: find-codeql
+        uses: github/codeql-action/init@esbena/ql
+        with:
+          languages: javascript # does not matter
+      - name: Build query pack
+        run: |
+          cd ql/src
+          "${CODEQL}" pack create
+          cd .codeql/pack/codeql/ql-all/0.0.0
+          zip "${PACKZIP}" -r .
+        env:
+          CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
+          PACKZIP: ${{ runner.temp }}/query-pack.zip
+      - name: Upload query pack
+        uses: actions/upload-artifact@v2
+        with:
+          name: query-pack-zip
+          path: ${{ runner.temp }}/query-pack.zip
+
+  extractors:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-latest]
-        #os: [ubuntu-latest, macos-latest, windows-latest]
+        os: ${{ fromJson(inputs.os || '["ubuntu-latest", "macos-latest", "windows-latest"]') }}
 
     runs-on: ${{ matrix.os }}
 
@@ -44,16 +73,6 @@ jobs:
       - name: Generate dbscheme
         if: ${{ matrix.os == 'ubuntu-latest' }}
         run: target/release/ql-generator --dbscheme ql/src/ql.dbscheme --library ql/src/codeql_ql/ast/internal/TreeSitter.qll
-      - uses: actions/upload-artifact@v2
-        if: ${{ matrix.os == 'ubuntu-latest' }}
-        with:
-          name: ql.dbscheme
-          path: ql/src/ql.dbscheme
-      - uses: actions/upload-artifact@v2
-        if: ${{ matrix.os == 'ubuntu-latest' }}
-        with:
-          name: TreeSitter.qll
-          path: ql/src/codeql_ql/ast/internal/TreeSitter.qll
       - uses: actions/upload-artifact@v2
         with:
           name: extractor-${{ matrix.os }}
@@ -63,33 +82,49 @@ jobs:
           retention-days: 1
   package:
     runs-on: ubuntu-latest
-    needs: build
+
+    needs:
+      - extractors
+      - queries
+
     steps:
       - uses: actions/checkout@v2
       - uses: actions/download-artifact@v2
         with:
-          name: ql.dbscheme
-          path: ql
+          name: query-pack-zip
+          path: query-pack-zip
       - uses: actions/download-artifact@v2
+        if: ${{ contains(fromJson(inputs.os || '["ubuntu-latest", "macos-latest", "windows-latest"]'), 'ubuntu-latest') }}
         with:
           name: extractor-ubuntu-latest
           path: linux64
-#      - uses: actions/download-artifact@v2
-#        with:
-#          name: extractor-windows-latest
-#          path: win64
       - uses: actions/download-artifact@v2
+        if: ${{ contains(fromJson(inputs.os || '["ubuntu-latest", "macos-latest", "windows-latest"]'), 'windows-latest') }}
+        with:
+         name: extractor-windows-latest
+         path: win64
+      - uses: actions/download-artifact@v2
+        if: ${{ contains(fromJson(inputs.os || '["ubuntu-latest", "macos-latest", "windows-latest"]'), 'macos-latest') }}
         with:
           name: extractor-macos-latest
           path: osx64
       - run: |
-          mkdir -p ql
-          cp -r codeql-extractor.yml tools ql/src/ql.dbscheme.stats ql/
-          mkdir -p ql/tools/{linux64,osx64}
-          cp linux64/ql-extractor ql/tools/linux64/extractor
-          cp osx64/ql-extractor ql/tools/osx64/extractor
-          chmod +x ql/tools/{linux64,osx64}/extractor
-          zip -rq codeql-ql.zip ql
+          unzip query-pack-zip/*.zip -d pack
+          cp -r codeql-extractor.yml tools ql/src/ql.dbscheme.stats pack/
+          mkdir -p pack/tools/{linux64,osx64,win64}
+          if [[ -f linux64/ql-extractor ]]; then
+            cp linux64/ql-extractor pack/tools/linux64/extractor
+            chmod +x pack/tools/linux64/extractor
+          fi
+          if [[ -f osx64/ql-extractor ]]; then
+            cp osx64/ql-extractor pack/tools/osx64/extractor
+            chmod +x pack/tools/osx64/extractor
+          fi
+          if [[ -f win64/ql-extractor.exe ]]; then
+            cp win64/ql-extractor.exe pack/tools/win64/extractor
+          fi
+          cd pack
+          zip -rq ../codeql-ql.zip .
       - uses: actions/upload-artifact@v2
         with:
           name: codeql-ql-pack

.github/workflows/nightly-changes.yml

@@ -0,0 +1,100 @@
+name: "Nightly analysis of changes in standard repos"
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '15 4 * * *'
+
+jobs:
+
+  build:
+    uses: github/codeql-ql/.github/workflows/build.yml@esbena/workflow-improvements
+    with:
+      os: '[ "ubuntu-latest" ]'
+
+  prepare-alert-branch:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code-scanning alert branch
+        uses: actions/checkout@v2
+        with:
+          ref: nightly-changes-alerts
+
+      - name: Checkout codeql
+        uses: actions/checkout@v2
+        with:
+          repository: github/codeql
+          path: codeql
+
+      - name: Checkout codeql-go
+        uses: actions/checkout@v2
+        with:
+          repository: github/codeql-go
+          path: codeql-go
+
+      - name: Store relevant files
+        run: |
+          git config --global user.name "${GITHUB_ACTOR}"
+          git config --global user.email "${GITHUB_ACTOR}+github/codeql-ql@users.noreply.github.com"
+
+          # see repo-tests/import-repositories.sh
+          for repo in codeql codeql-go; do
+            git -C "$repo" rev-parse HEAD > "$repo.txt";
+            # remove upgrades and tests (heuristic)
+            find "$repo" -depth -type d \( -path "*/upgrades" -o -path "*/ql/test" \) -exec rm -rf {} \; ;
+            # only preserve files mentioned in tools/autobuild.sh
+            find "$repo" -type f -not \( -name "*.qll" -o -name "*.ql" -o -name "*.dbscheme" -o -name qlpack.yml \) -exec rm -f {} \; ;
+            # remove empty directories (git does not care though)
+            find "$repo" -type d -empty -delete;
+            git add "$repo" "$repo.txt";
+            git commit --allow-empty -m "Add $repo sources ($(tr -d '\n' < $repo.txt))";
+          done
+
+          git push
+
+  analyze:
+    name: Analyze
+    needs:
+      - build
+      - prepare-alert-branch
+
+    runs-on: ubuntu-latest
+
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    steps:
+      - name: Download pack
+        uses: actions/download-artifact@v2
+        with:
+          name: codeql-ql-pack
+          path: ${{ runner.temp }}/codeql-ql-pack-artifact
+
+      - name: Prepare pack
+        run: |
+          unzip "${PACK_ARTIFACT}/*.zip" -d "${PACK}"
+        env:
+          PACK_ARTIFACT: ${{ runner.temp }}/codeql-ql-pack-artifact
+          PACK: ${{ runner.temp }}/pack
+
+      - name: Hack codeql-action options
+        run: |
+          JSON=$(jq -nc --arg pack "${PACK}" '.resolve.queries=["--search-path", $pack] | .resolve.extractor=["--search-path", $pack] | .database.init=["--search-path", $pack]')
+          echo "CODEQL_ACTION_EXTRA_OPTIONS=${JSON}" >> ${GITHUB_ENV}
+        env:
+          PACK: ${{ runner.temp }}/pack
+
+      - name: Checkout code-scanning alert branch
+        uses: actions/checkout@v2
+        with:
+          ref: nightly-changes-alerts
+
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@esbena/ql
+        with:
+          languages: ql
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@esbena/ql