codeql/.github/workflows/java-model-diff.yml

name: Models as Data

on:
  workflow_dispatch:
    inputs:
      projects:
        description: "The projects to generate models for"
        required: true
        default: '["netty/netty"]'
  pull_request:
    branches:
      - main
    paths:
      - "java/ql/src/utils/model-generator/**/*.*"

jobs:
  model-diff:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        # large
        # ["google/guava", "spring-projects/spring-framework", "apache/poi"]
        # others
        # ["FasterXML/jackson-core", "FasterXML/jackson-databind", "google/gson", "JodaOrg/joda-time"
        slug: ${{fromJson(github.event.inputs.projects || '["apache/commons-codec", "apache/commons-io", "apache/commons-beanutils", "apache/commons-logging", "apache/commons-fileupload", "apache/commons-lang", "apache/commons-validator", "apache/commons-csv", "apache/dubbo"]' )}}
    steps:
      - name: Clone self (github/codeql) for baseline
        uses: actions/checkout@v2
        with:
          path: codeql-baseline
          ref: ${{ github.base_ref }}
      - name: Clone self (github/codeql) with new generator
        uses: actions/checkout@v2
        with:
          path: codeql-head
          ref: ${{ github.ref }}
      - uses: ./codeql-baseline/.github/actions/fetch-codeql
      - name: Download database
        run: |
          set -x
          mkdir lib-dbs
          SHORTNAME=${SLUG//[^a-zA-Z0-9_]/}
          projectId=`curl -s https://lgtm.com/api/v1.0/projects/g/${matrix.slugs}} | jq .id`
          curl -L "https://lgtm.com/api/v1.0/snapshots/$projectId/java" -o $SHORTNAME.zip
          unzip -q -d $SHORTNAME-db $SHORTNAME.zip
          mkdir lib-dbs/$SHORTNAME/
          mv $SHORTNAME-db/`ls -1 $SHORTNAME-db`/* lib-dbs/$SHORTNAME/
      - name: Generate Models (Baseline and HEAD)
        run: |
          set -x
          mkdir tmp-models
          MODELS=`pwd`/tmp-models
          MODE="baseline"
          DATABASES=`pwd`/lib-dbs

          analyzeDatabaseWithCheckout() {
            QL_VARIANT=$1
            DATABASE=$2
            cd codeql-$QL_VARIANT
            SHORTNAME=`basename $DATABASE`
            python java/ql/src/utils/model-generator/GenerateFlowModel.py $DATABASE $MODELS/${SHORTNAME}.qll
            mv $MODELS/${SHORTNAME}.qll $MODELS/${SHORTNAME}_${QL_VARIANT}.qll
            cd ..
          }

          for d in $DATABASES/*/ ; do
            ls -1 "$d"

            analyzeDatabaseWithCheckout "baseline" $d
            if [[ "$s1" != "$s2" ]]
            then
              analyzeDatabaseWithCheckout "head" $d
            fi
          done
      - name: Install diff2html
        run: |
          npm install -g diff2html-cli
      - name: Generate Model Diff
        run: |
          set -x
          if [[ "$s1" == "$s2" ]]
          then
            echo "Skipping diff generation as github.base_ref and github.ref are the same"
          else
            MODELS=`pwd`/tmp-models
            ls -1 tmp-models/
            for m in $MODELS/*_baseline.qll ; do
              t="${m/baseline/"head"}"
              basename=`basename $m`
              name="diff_${basename/_baseline.qll/""}"
              (diff -w -u $m $t | diff2html -i stdin -F $MODELS/$name.html) || true
            done
          fi
      - uses: actions/upload-artifact@v2
        with:
          name: models
          path: tmp-models/*.qll
          retention-days: 20
      - uses: actions/upload-artifact@v2
        with:
          name: diffs
          path: tmp-models/*.html
          retention-days: 20