hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2025-12-21 03:06:31 +01:00
Code Issues Packages Projects Releases Wiki Activity

Test case generator: improve error reporting

Browse Source 
We now distinguish cases where SSV rows are not in scope at all from those where they don't identify a known type or method, or where input or output specs could not be parsed.
This commit is contained in:
Chris Smowton
2021-06-21 19:46:40 +01:00
parent dff9c717bc
commit 59725d635b
2 changed files with 55 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
java/ql/src/utils/GenerateFlowTestCase.py
View File

@@ -15,7 +15,7 @@ if any(s == "--help" for s in sys.argv):
print("""Usage:
GenerateFlowTestCase.py specsToTest.ssv projectPom.xml outdir
This generates test cases exercising taint flow specifications found in specsToTest.ssv
This generates test cases exercising function model specifications found in specsToTest.ssv
producing files Test.java, test.ql and test.expected in outdir.
projectPom.xml should be a Maven pom sufficient to resolve the classes named in specsToTest.ssv.
@@ -115,11 +115,6 @@ with open(qlFile, "w") as f:
f.write("import java\nimport utils.GenerateFlowTestCase\n\nclass GenRow extends CsvRow {\n\n\tGenRow() {\n\t\tthis = [\n")
f.write(",\n".join('\t\t\t"%s"' % spec.strip() for spec in specs))
f.write("\n\t\t]\n\t}\n}\n")
f.write("""
query string getAFailedRow() {
result = any(GenRow row | not exists(RowTestSnippet r | exists(r.getATestSnippetForRow(row))) | row)
}
""")
print("Generating tests")
generatedBqrs = os.path.join(queryDir, "out.bqrs")
@@ -144,17 +139,24 @@ def getTuples(queryName, jsonResult, fname):
with open(generatedJson, "r") as f:
generateOutput = json.load(f)
testCaseRows = getTuples("getTestCase", generateOutput, generatedJson)
supportModelRows = getTuples("getASupportMethodModel", generateOutput, generatedJson)
failedRows = getTuples("getAFailedRow", generateOutput, generatedJson)
expectedTables = ("getTestCase", "getASupportMethodModel", "missingSummaryModelCsv", "getAParseFailure")
testCaseRows, supportModelRows, missingSummaryModelCsvRows, parseFailureRows = \
tuple([getTuples(k, generateOutput, generatedJson) for k in expectedTables])
if len(testCaseRows) != 1 or len(testCaseRows[0]) != 1:
print("Expected exactly one getTestCase result with one column (got: %s)" % json.dumps(testCaseRows), file = sys.stderr)
if any(len(row) != 1 for row in supportModelRows):
print("Expected exactly one column in getASupportMethodModel relation (got: %s)" % json.dumps(supportModelRows), file = sys.stderr)
if any(len(row) != 1 for row in failedRows):
print("Expected exactly one column in getAFailedRow relation (got: %s)" % json.dumps(failedRows), file = sys.stderr)
if len(failedRows) != 0:
print("The following rows failed to generate any test case. Check package, class and method name spelling, and argument and result specifications:\n%s" % "\n".join(r[0] for r in failedRows), file = sys.stderr)
if any(len(row) != 2 for row in parseFailureRows):
print("Expected exactly two columns in parseFailureRows relation (got: %s)" % json.dumps(parseFailureRows), file = sys.stderr)
if len(missingSummaryModelCsvRows) != 0:
print("Tests for some SSV rows were requested that were not in scope (SummaryModelCsv.row does not hold):\n" + "\n".join(r[0] for r in missingSummaryModelCsvRows))
sys.exit(1)
if len(parseFailureRows) != 0:
print("The following rows failed to generate any test case. Check package, class and method name spelling, and argument and result specifications:\n%s" % "\n".join(r[0] + ": " + r[1] for r in parseFailureRows), file = sys.stderr)
sys.exit(1)
with open(resultJava, "w") as f:
f.write(generateOutput["getTestCase"]["tuples"][0][0])