diff --git a/.github/workflows/mad_modelDiff.yml b/.github/workflows/mad_modelDiff.yml new file mode 100644 index 00000000000..a4b9169b44e --- /dev/null +++ b/.github/workflows/mad_modelDiff.yml @@ -0,0 +1,103 @@ +name: Models as Data - Diff + +on: + workflow_dispatch: + inputs: + projects: + description: "The projects to generate models for" + required: true + default: '["netty/netty"]' + pull_request: + branches: + - main + paths: + - "java/ql/src/utils/model-generator/**/*.*" + - ".github/workflows/mad_modelDiff.yml" + +permissions: + contents: read + +jobs: + model-diff: + name: Model Difference + runs-on: ubuntu-latest + if: github.repository == 'github/codeql' + strategy: + matrix: + slug: ${{fromJson(github.event.inputs.projects || '["apache/commons-codec", "apache/commons-io", "apache/commons-beanutils", "apache/commons-logging", "apache/commons-fileupload", "apache/commons-lang", "apache/commons-validator", "apache/commons-csv", "apache/dubbo"]' )}} + steps: + - name: Clone github/codeql from PR + uses: actions/checkout@v2 + if: github.event.pull_request + with: + path: codeql-pr + - name: Clone github/codeql from main + uses: actions/checkout@v2 + with: + path: codeql-main + ref: main + - uses: ./codeql-main/.github/actions/fetch-codeql + - name: Download database + env: + SLUG: ${{ matrix.slug }} + run: | + set -x + mkdir lib-dbs + SHORTNAME=${SLUG//[^a-zA-Z0-9_]/} + projectId=`curl -s https://lgtm.com/api/v1.0/projects/g/${SLUG} | jq .id` + curl -L "https://lgtm.com/api/v1.0/snapshots/$projectId/java" -o "$SHORTNAME.zip" + unzip -q -d "${SHORTNAME}-db" "${SHORTNAME}.zip" + mkdir "lib-dbs/$SHORTNAME/" + mv "${SHORTNAME}-db/"$(ls -1 "${SHORTNAME}"-db)/* "lib-dbs/${SHORTNAME}/" + - name: Generate Models (PR and main) + run: | + set -x + mkdir tmp-models + MODELS=`pwd`/tmp-models + DATABASES=`pwd`/lib-dbs + + analyzeDatabaseWithCheckout() { + QL_VARIANT=$1 + DATABASE=$2 + cd codeql-$QL_VARIANT + SHORTNAME=`basename $DATABASE` + python java/ql/src/utils/model-generator/GenerateFlowModel.py $DATABASE $MODELS/${SHORTNAME}.qll + mv $MODELS/${SHORTNAME}.qll $MODELS/${SHORTNAME}Generated_${QL_VARIANT}.qll + cd .. + } + + for d in $DATABASES/*/ ; do + ls -1 "$d" + + analyzeDatabaseWithCheckout "main" $d + if [[ "$GITHUB_EVENT_NAME" == "pull_request" ]] + then + analyzeDatabaseWithCheckout "pr" $d + fi + done + - name: Install diff2html + if: github.event.pull_request + run: | + npm install -g diff2html-cli + - name: Generate Model Diff + if: github.event.pull_request + run: | + set -x + MODELS=`pwd`/tmp-models + ls -1 tmp-models/ + for m in $MODELS/*_main.qll ; do + t="${m/main/"pr"}" + basename=`basename $m` + name="diff_${basename/_main.qll/""}" + (diff -w -u $m $t | diff2html -i stdin -F $MODELS/$name.html) || true + done + - uses: actions/upload-artifact@v2 + with: + name: models + path: tmp-models/*.qll + retention-days: 20 + - uses: actions/upload-artifact@v2 + with: + name: diffs + path: tmp-models/*.html + retention-days: 20