Add comparison step to workflow

.github/workflows/csv-coverage.yml

@@ -1,4 +1,4 @@
-name: Build CSV flow coverage report
+name: Build/check CSV flow coverage report
 
 on:
   workflow_dispatch:
@@ -13,10 +13,14 @@ on:
   pull_request:
     paths:
       - '.github/workflows/csv-coverage.yml'
-      - 'misc/scripts/generate-csv-coverage-report.py'
       - 'java/ql/src/meta/frameworks/Coverage.ql'
-      - 'java/documentation/library-coverage/cwe-sink.csv'
-      - 'java/documentation/library-coverage/frameworks.csv'
+      - 'misc/scripts/library-coverage/*.py'
+      # input data files
+      - '*/documentation/library-coverage/cwe-sink.csv'
+      - '*/documentation/library-coverage/frameworks.csv'
+      # coverage report files
+      - '*/documentation/library-coverage/flow-model-coverage.csv'
+      - '*/documentation/library-coverage/flow-model-coverage.rst'
 
 jobs:
   build:
@@ -54,7 +58,7 @@ jobs:
       run: unzip -d codeql-cli codeql-linux64.zip
     - name: Build modeled package list
       run: |
-        PATH="$PATH:codeql-cli/codeql" python script/misc/scripts/generate-csv-coverage-report.py ci codeqlModels script
+        PATH="$PATH:codeql-cli/codeql" python script/misc/scripts/library-coverage/generate-report.py ci codeqlModels script
     - name: Upload CSV package list
       uses: actions/upload-artifact@v2
       with:
@@ -65,4 +69,8 @@ jobs:
       with:
         name: rst-flow-model-coverage
         path: flow-model-coverage-*.rst
+    - name: Check coverage files
+      if: github.event.pull_request
+      run: |
+        python script/misc/scripts/library-coverage/compare-files.py codeqlModels
 

misc/scripts/library-coverage/compare-files.py

@@ -0,0 +1,38 @@
+import sys
+import os
+import settings
+import filecmp
+
+"""
+This script compares the generated CSV coverage files with the ones in the codebase.
+"""
+
+
+def check_file_exists(file):
+    if not os.path.exists(file):
+        print("Expected file '" + file + "' doesn't exist.", file=sys.stderr)
+        sys.exit(1)
+
+
+languages = ['java']
+
+for lang in languages:
+    repo_output_rst = settings.repo_output_rst.format(language=lang)
+    repo_output_csv = settings.repo_output_csv.format(language=lang)
+
+    generated_output_rst = settings.generated_output_rst.format(language=lang)
+    generated_output_csv = settings.generated_output_csv.format(language=lang)
+
+    check_file_exists(repo_output_rst)
+    check_file_exists(repo_output_csv)
+    check_file_exists(generated_output_rst)
+    check_file_exists(generated_output_csv)
+
+    filecmp.clear_cache()
+    if not filecmp.cmp(repo_output_rst, generated_output_rst, shallow=False) or not filecmp.cmp(repo_output_csv, generated_output_csv, shallow=False):
+        print("Error: The generated files for '" + lang +
+              "' do not match the ones in the codebase. Please check and fix.", file=sys.stderr)
+        sys.exit(1)
+
+    print("The generated files for '" + lang +
+          "' match the ones in the codebase.")

misc/scripts/library-coverage/generate-report.py

@@ -3,6 +3,7 @@ import csv
 import sys
 import os
 import shutil
+import settings
 
 """
 This script runs the CSV coverage report QL query, and transforms it to a more readable format.
@@ -135,13 +136,12 @@ if mode != "dev" and mode != "ci":
           ". Expected either 'dev' or 'ci'.", file=sys.stderr)
     exit(1)
 
+# The QL model holding the CSV info can come from directly a PR or the main branch, but optionally we can use an earlier
+# SHA too, therefore it's checked out seperately into a dedicated subfolder.
 query_prefix = ""
-data_prefix = ""
 if len(sys.argv) > 2:
     query_prefix = sys.argv[2] + "/"
 
-if len(sys.argv) > 3:
-    data_prefix = sys.argv[3] + "/"
 
 # Languages for which we want to generate coverage reports.
 configs = [
@@ -150,17 +150,16 @@ configs = [
 ]
 
 # The names of input and output files. The placeholder {language} is replaced with the language name.
-documentation_folder = "{language}/documentation/library-coverage/"
 output_ql_csv = "output-{language}.csv"
-input_framework_csv = data_prefix + documentation_folder + "frameworks.csv"
-input_cwe_sink_csv = data_prefix + documentation_folder + "cwe-sink.csv"
+input_framework_csv = settings.documentation_folder + "frameworks.csv"
+input_cwe_sink_csv = settings.documentation_folder + "cwe-sink.csv"
 
 if mode == "dev":
-    output_rst = data_prefix + documentation_folder + "flow-model-coverage.rst"
-    output_csv = data_prefix + documentation_folder + "flow-model-coverage.csv"
+    output_rst = settings.repo_output_rst
+    output_csv = settings.repo_output_csv
 else:
-    output_rst = "flow-model-coverage-{language}.rst"
-    output_csv = "flow-model-coverage-{language}.csv"
+    output_rst = settings.generated_output_rst
+    output_csv = settings.generated_output_csv
 
 for config in configs:
     lang = config.lang

misc/scripts/library-coverage/settings.py

@@ -0,0 +1,23 @@
+import sys
+
+
+generated_output_rst = "flow-model-coverage-{language}.rst"
+generated_output_csv = "flow-model-coverage-{language}.csv"
+
+
+# The CI job checks out the codebase to a subfolder
+data_prefix = ""
+
+index = 1
+if sys.argv[0].endswith("generate-report.py"):
+    index = 3
+
+if len(sys.argv) > index:
+    data_prefix = sys.argv[index] + "/"
+
+
+documentation_folder = data_prefix + \
+    "{language}/documentation/library-coverage/"
+
+repo_output_rst = documentation_folder + "flow-model-coverage.rst"
+repo_output_csv = documentation_folder + "flow-model-coverage.csv"