mirror of
https://github.com/github/codeql.git
synced 2025-12-17 01:03:14 +01:00
Add CSV coverage PR commenter
This commit is contained in:
Tamas Vajk
183
misc/scripts/library-coverage/compare-files-comment-pr.py
Normal file
183
misc/scripts/library-coverage/compare-files-comment-pr.py
Normal file
@@ -0,0 +1,183 @@
|
||||
import sys
|
||||
import os
|
||||
import settings
|
||||
import difflib
|
||||
import utils
|
||||
|
||||
"""
|
||||
This script compares the generated CSV coverage files with the ones in the codebase.
|
||||
"""
|
||||
|
||||
|
||||
def check_file_exists(file):
|
||||
if not os.path.exists(file):
|
||||
print("Expected file '" + file + "' doesn't exist.", file=sys.stderr)
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def ignore_line_ending(ch):
|
||||
return difflib.IS_CHARACTER_JUNK(ch, ws=" \r\n")
|
||||
|
||||
|
||||
def compare_files(file1, file2):
|
||||
messages = compare_files_str(file1, file2)
|
||||
if messages == "":
|
||||
return True
|
||||
|
||||
print(messages, end="", file=sys.stderr)
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def compare_files_str(file1, file2):
|
||||
diff = difflib.ndiff(open(file1).readlines(),
|
||||
open(file2).readlines(), None, ignore_line_ending)
|
||||
ret = ""
|
||||
for line in diff:
|
||||
if line.startswith("+") or line.startswith("-"):
|
||||
ret += line
|
||||
|
||||
return ret
|
||||
|
||||
|
||||
def comment_pr(folder1, folder2, output_file, repo, pr_number, run_id):
|
||||
compare_folders(folder1, folder2, output_file)
|
||||
size = os.path.getsize(output_file)
|
||||
if size == 0:
|
||||
print("No difference in the coverage reports")
|
||||
return
|
||||
|
||||
comment = ":warning: The head of this PR and the base branch were compared for differences in the framework coverage reports. " + \
|
||||
"The generated reports are available in the [artifacts of this workflow run](https://github.com/" + repo + "/actions/runs/" + run_id + "). " + \
|
||||
"The differences will be picked up by the nightly job after the PR gets merged. "
|
||||
|
||||
if size < 2000:
|
||||
print("There's a small change in the CSV framework coverage reports")
|
||||
comment += "The following differences were found: \n\n"
|
||||
with open(output_file, 'r') as file:
|
||||
comment += file.read()
|
||||
else:
|
||||
print("There's a large change in the CSV framework coverage reports")
|
||||
comment += "The differences can be found in the " + \
|
||||
output_file + " artifact of this job."
|
||||
|
||||
post_comment(comment, repo, pr_number)
|
||||
|
||||
|
||||
def post_comment(comment, repo, pr_number):
|
||||
print("Posting comment to PR #" + str(pr_number))
|
||||
utils.subprocess_run(["gh", "pr", "comment", pr_number,
|
||||
"--repo", repo, "--body", comment])
|
||||
|
||||
|
||||
def compare_folders(folder1, folder2, output_file):
|
||||
languages = ['java']
|
||||
|
||||
return_md = ""
|
||||
|
||||
for lang in languages:
|
||||
expected_files = ""
|
||||
|
||||
generated_output_rst = settings.generated_output_rst.format(
|
||||
language=lang)
|
||||
generated_output_csv = settings.generated_output_csv.format(
|
||||
language=lang)
|
||||
|
||||
# check if files exist in both folder1 and folder 2
|
||||
if not check_file_exists(folder1 + "/" + generated_output_rst):
|
||||
expected_files += "- " + generated_output_rst + \
|
||||
" doesn't exist in folder " + folder1 + "\n"
|
||||
if not check_file_exists(folder2 + "/" + generated_output_rst):
|
||||
expected_files += "- " + generated_output_rst + \
|
||||
" doesn't exist in folder " + folder2 + "\n"
|
||||
if not check_file_exists(folder1 + "/" + generated_output_csv):
|
||||
expected_files += "- " + generated_output_csv + \
|
||||
" doesn't exist in folder " + folder1 + "\n"
|
||||
if not check_file_exists(folder2 + "/" + generated_output_csv):
|
||||
expected_files += "- " + generated_output_csv + \
|
||||
" doesn't exist in folder " + folder2 + "\n"
|
||||
|
||||
if expected_files != "":
|
||||
print("Expected files are missing", file=sys.stderr)
|
||||
return_md += "\n### " + lang + "\n\n#### Expected files are missing for " + \
|
||||
lang + "\n" + expected_files + "\n"
|
||||
continue
|
||||
|
||||
# compare contents of files
|
||||
cmp1 = compare_files_str(
|
||||
folder1 + "/" + generated_output_rst, folder2 + "/" + generated_output_rst)
|
||||
cmp2 = compare_files_str(
|
||||
folder1 + "/" + generated_output_csv, folder2 + "/" + generated_output_csv)
|
||||
|
||||
if cmp1 != "" or cmp2 != "":
|
||||
print("Generated file contents are not matching", file=sys.stderr)
|
||||
return_md += "\n### " + lang + "\n\n#### Generated file changes for " + \
|
||||
lang + "\n\n"
|
||||
if cmp1 != "":
|
||||
return_md += "- Changes to " + generated_output_rst + \
|
||||
":\n```diff\n" + cmp1 + "```\n\n"
|
||||
if cmp2 != "":
|
||||
return_md += "- Changes to " + generated_output_csv + \
|
||||
":\n```diff\n" + cmp2 + "```\n\n"
|
||||
|
||||
with open(output_file, 'w', newline='') as out:
|
||||
out.write(return_md)
|
||||
|
||||
|
||||
comment_pr(sys.argv[1], sys.argv[2], sys.argv[3],
|
||||
sys.argv[4], sys.argv[5], sys.argv[6])
|
||||
|
||||
|
||||
# def compare_generated_and_repo_files():
|
||||
# languages = ['java']
|
||||
|
||||
# all_ok = True
|
||||
|
||||
# for lang in languages:
|
||||
# repo_output_rst = settings.repo_output_rst.format(language=lang)
|
||||
# repo_output_csv = settings.repo_output_csv.format(language=lang)
|
||||
|
||||
# generated_output_rst = settings.generated_output_rst.format(
|
||||
# language=lang)
|
||||
# generated_output_csv = settings.generated_output_csv.format(
|
||||
# language=lang)
|
||||
|
||||
# exists = check_file_exists(repo_output_rst)
|
||||
# if not exists:
|
||||
# sys.exit(1)
|
||||
|
||||
# exists = check_file_exists(repo_output_csv)
|
||||
# if not exists:
|
||||
# sys.exit(1)
|
||||
|
||||
# exists = check_file_exists(generated_output_rst)
|
||||
# if not exists:
|
||||
# sys.exit(1)
|
||||
|
||||
# exists = check_file_exists(generated_output_csv)
|
||||
# if not exists:
|
||||
# sys.exit(1)
|
||||
|
||||
# docs_folder = settings.documentation_folder_no_prefix.format(
|
||||
# language=lang)
|
||||
|
||||
# rst_ok = compare_files(repo_output_rst, generated_output_rst)
|
||||
# if not rst_ok:
|
||||
# print("The generated file doesn't match the one in the codebase. Please check and fix file '" +
|
||||
# docs_folder + settings.output_rst_file_name + "'.", file=sys.stderr)
|
||||
# csv_ok = compare_files(repo_output_csv, generated_output_csv)
|
||||
# if not csv_ok:
|
||||
# print("The generated file doesn't match the one in the codebase. Please check and fix file '" +
|
||||
# docs_folder + settings.output_csv_file_name + "'.", file=sys.stderr)
|
||||
|
||||
# if not rst_ok or not csv_ok:
|
||||
# print("The generated CSV coverage report files for '" + lang + "' don't match the ones in the codebase. Please update the files in '" +
|
||||
# docs_folder + "'. The new files can be downloaded from the artifacts of this job.", file=sys.stderr)
|
||||
# all_ok = False
|
||||
# else:
|
||||
# print("The generated files for '" + lang +
|
||||
# "' match the ones in the codebase.")
|
||||
|
||||
# if not all_ok:
|
||||
# sys.exit(1)
|
||||
@@ -1,69 +0,0 @@
|
||||
import sys
|
||||
import os
|
||||
import settings
|
||||
import difflib
|
||||
|
||||
"""
|
||||
This script compares the generated CSV coverage files with the ones in the codebase.
|
||||
"""
|
||||
|
||||
|
||||
def check_file_exists(file):
|
||||
if not os.path.exists(file):
|
||||
print("Expected file '" + file + "' doesn't exist.", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def ignore_line_ending(ch):
|
||||
return difflib.IS_CHARACTER_JUNK(ch, ws=" \r\n")
|
||||
|
||||
|
||||
def compare_files(file1, file2, path_to_report):
|
||||
has_differences = False
|
||||
diff = difflib.ndiff(open(file1).readlines(),
|
||||
open(file2).readlines(), None, ignore_line_ending)
|
||||
for line in diff:
|
||||
if line.startswith("+") or line.startswith("-"):
|
||||
print(line, end="", file=sys.stderr)
|
||||
has_differences = True
|
||||
|
||||
if has_differences:
|
||||
print("The generated file doesn't match the one in the codebase. Please check and fix file '" +
|
||||
path_to_report + "'.", file=sys.stderr)
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
languages = ['java']
|
||||
|
||||
all_ok = True
|
||||
|
||||
for lang in languages:
|
||||
repo_output_rst = settings.repo_output_rst.format(language=lang)
|
||||
repo_output_csv = settings.repo_output_csv.format(language=lang)
|
||||
|
||||
generated_output_rst = settings.generated_output_rst.format(language=lang)
|
||||
generated_output_csv = settings.generated_output_csv.format(language=lang)
|
||||
|
||||
check_file_exists(repo_output_rst)
|
||||
check_file_exists(repo_output_csv)
|
||||
check_file_exists(generated_output_rst)
|
||||
check_file_exists(generated_output_csv)
|
||||
|
||||
docs_folder = settings.documentation_folder_no_prefix.format(language=lang)
|
||||
|
||||
rst_ok = compare_files(repo_output_rst, generated_output_rst,
|
||||
docs_folder + settings.output_rst_file_name)
|
||||
csv_ok = compare_files(repo_output_csv, generated_output_csv,
|
||||
docs_folder + settings.output_csv_file_name)
|
||||
|
||||
if not rst_ok or not csv_ok:
|
||||
print("The generated CSV coverage report files for '" + lang + "' don't match the ones in the codebase. Please update the files in '" +
|
||||
docs_folder + "'. The new files can be downloaded from the artifacts of this job.", file=sys.stderr)
|
||||
all_ok = False
|
||||
else:
|
||||
print("The generated files for '" + lang +
|
||||
"' match the ones in the codebase.")
|
||||
|
||||
if not all_ok:
|
||||
sys.exit(1)
|
||||
@@ -2,10 +2,12 @@ import subprocess
|
||||
import os
|
||||
import csv
|
||||
import sys
|
||||
import shlex
|
||||
|
||||
|
||||
def subprocess_run(cmd):
|
||||
"""Runs a command through subprocess.run, with a few tweaks. Raises an Exception if exit code != 0."""
|
||||
print(shlex.join(cmd))
|
||||
return subprocess.run(cmd, capture_output=True, text=True, env=os.environ.copy(), check=True)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user