mirror of
https://github.com/github/codeql.git
synced 2026-04-29 02:35:15 +02:00
Add flow-through test case generator
This commit is contained in:
Chris Smowton
144
java/ql/src/utils/GenerateFlowTestCase.py
Normal file
144
java/ql/src/utils/GenerateFlowTestCase.py
Normal file
@@ -0,0 +1,144 @@
|
||||
import errno
|
||||
import json
|
||||
import os
|
||||
import os.path
|
||||
import re
|
||||
import shlex
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
if len(sys.argv) != 4:
|
||||
print("Usage: GenerateFlowTestCase.py specsToTest.ssv projectPom.xml outdir", file=sys.stderr)
|
||||
print("specsToTest.ssv should contain SSV rows describing method taint-propagation specifications to test", file=sys.stderr)
|
||||
print("projectPom.xml should import dependencies sufficient to resolve the types used in specsToTest.ssv", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
os.makedirs(sys.argv[3])
|
||||
except Exception as e:
|
||||
if e.errno != errno.EEXIST:
|
||||
print("Failed to create output directory %s: %s" % (sys.argv[3], e))
|
||||
sys.exit(1)
|
||||
|
||||
resultJava = os.path.join(sys.argv[3], "Test.java")
|
||||
resultQl = os.path.join(sys.argv[3], "test.ql")
|
||||
|
||||
if os.path.exists(resultJava) or os.path.exists(resultQl):
|
||||
print("Won't overwrite existing files '%s' or '%s'" % (resultJava, resultQl), file = sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
workDir = tempfile.mkdtemp()
|
||||
|
||||
# Step 1: make a database that touches all types whose methods we want to test:
|
||||
print("Creating Maven project")
|
||||
projectDir = os.path.join(workDir, "mavenProject")
|
||||
os.makedirs(projectDir)
|
||||
|
||||
try:
|
||||
shutil.copyfile(sys.argv[2], os.path.join(projectDir, "pom.xml"))
|
||||
except Exception as e:
|
||||
print("Failed to read project POM %s: %s" % (sys.argv[2], e), file = sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
commentRegex = re.compile("^\s*(//|#)")
|
||||
def isComment(s):
|
||||
return commentRegex.match(s) is not None
|
||||
|
||||
try:
|
||||
with open(sys.argv[1], "r") as f:
|
||||
specs = [l for l in f if not isComment(l)]
|
||||
except Exception as e:
|
||||
print("Failed to open %s: %s\n" % (sys.argv[1], e))
|
||||
sys.exit(1)
|
||||
|
||||
projectTestPkgDir = os.path.join(projectDir, "src", "main", "java", "test")
|
||||
projectTestFile = os.path.join(projectTestPkgDir, "Test.java")
|
||||
|
||||
os.makedirs(projectTestPkgDir)
|
||||
|
||||
def qualifiedOuterNameFromSsvRow(row):
|
||||
cells = row.split(";")
|
||||
if len(cells) < 2:
|
||||
return None
|
||||
return cells[0] + "." + cells[1].replace("$", ".")
|
||||
|
||||
with open(projectTestFile, "w") as testJava:
|
||||
testJava.write("package test;\n\npublic class Test {\n\n")
|
||||
|
||||
for i, spec in enumerate(specs):
|
||||
outerName = qualifiedOuterNameFromSsvRow(spec)
|
||||
if outerName is None:
|
||||
print("A taint specification has the wrong format: should be 'package;classname;methodname....'", file = sys.stderr)
|
||||
print("Mis-formatted row: " + spec, file = sys.stderr)
|
||||
sys.exit(1)
|
||||
testJava.write("\t%s obj%d = null;\n" % (outerName, i))
|
||||
|
||||
testJava.write("}")
|
||||
|
||||
print("Creating project database")
|
||||
cmd = ["codeql", "database", "create", "--language=java", "db"]
|
||||
ret = subprocess.call(cmd, cwd = projectDir)
|
||||
if ret != 0:
|
||||
print("Failed to create project database. Check that '%s' is a valid POM that pulls in all necessary dependencies, and '%s' specifies valid classes and methods." % (sys.argv[2], sys.argv[1]), file = sys.stderr)
|
||||
print("Failed command was: %s (cwd: %s)" % (shlex.join(cmd), projectDir), file = sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
print("Creating test-generation query")
|
||||
queryDir = os.path.join(workDir, "query")
|
||||
os.makedirs(queryDir)
|
||||
qlFile = os.path.join(queryDir, "gen.ql")
|
||||
with open(os.path.join(queryDir, "qlpack.yml"), "w") as f:
|
||||
f.write("name: test-generation-query\nversion: 0.0.0\nlibraryPathDependencies: codeql-java")
|
||||
with open(qlFile, "w") as f:
|
||||
f.write("import java\nimport utils.GenerateFlowTestCase\n\nclass GenRow extends CsvRow {\n\n\tGenRow() {\n\t\tthis = [\n")
|
||||
f.write(",\n".join('\t\t\t"%s"' % spec.strip() for spec in specs))
|
||||
f.write("\n\t\t]\n\t}\n}\n")
|
||||
|
||||
print("Generating tests")
|
||||
generatedBqrs = os.path.join(queryDir, "out.bqrs")
|
||||
cmd = ['codeql', 'query', 'run', qlFile, '--database', os.path.join(projectDir, "db"), '--output', generatedBqrs]
|
||||
ret = subprocess.call(cmd)
|
||||
if ret != 0:
|
||||
print("Failed to generate tests. Failed command was: " + shlex.join(cmd))
|
||||
sys.exit(1)
|
||||
|
||||
generatedJson = os.path.join(queryDir, "out.json")
|
||||
cmd = ['codeql', 'bqrs', 'decode', generatedBqrs, '--format=json', '--output', generatedJson]
|
||||
ret = subprocess.call(cmd)
|
||||
if ret != 0:
|
||||
print("Failed to decode BQRS. Failed command was: " + shlex.join(cmd))
|
||||
sys.exit(1)
|
||||
|
||||
def getTuples(queryName, jsonResult, fname):
|
||||
if queryName not in jsonResult or "tuples" not in jsonResult[queryName]:
|
||||
print("Failed to read generated tests: expected key '%s' with a 'tuples' subkey in file '%s'" % (queryName, fname), file = sys.stderr)
|
||||
sys.exit(1)
|
||||
return jsonResult[queryName]["tuples"]
|
||||
|
||||
with open(generatedJson, "r") as f:
|
||||
generateOutput = json.load(f)
|
||||
testCaseRows = getTuples("getTestCase", generateOutput, generatedJson)
|
||||
supportModelRows = getTuples("getASupportMethodModel", generateOutput, generatedJson)
|
||||
if len(testCaseRows) != 1 or len(testCaseRows[0]) != 1:
|
||||
print("Expected exactly one getTestCase result with one column (got: %s)" % json.dumps(testCaseRows), file = sys.stderr)
|
||||
if any(len(row) != 1 for row in supportModelRows):
|
||||
print("Expected exactly one column in getASupportMethodModel relation (got: %s)" % json.dumps(supportModelRows), file = sys.stderr)
|
||||
|
||||
with open(resultJava, "w") as f:
|
||||
f.write(generateOutput["getTestCase"]["tuples"][0][0])
|
||||
|
||||
scriptPath = os.path.dirname(sys.argv[0])
|
||||
|
||||
with open(resultQl, "w") as f:
|
||||
with open(os.path.join(scriptPath, "testHeader.qlfrag"), "r") as header:
|
||||
shutil.copyfileobj(header, f)
|
||||
f.write(", ".join('"%s"' % modelSpecRow[0].strip() for modelSpecRow in supportModelRows))
|
||||
with open(os.path.join(scriptPath, "testFooter.qlfrag"), "r") as header:
|
||||
shutil.copyfileobj(header, f)
|
||||
|
||||
cmd = ['codeql', 'query', 'format', '-qq', '-i', resultQl]
|
||||
subprocess.call(cmd)
|
||||
|
||||
shutil.rmtree(workDir)
|
||||
295
java/ql/src/utils/GenerateFlowTestCase.qll
Normal file
295
java/ql/src/utils/GenerateFlowTestCase.qll
Normal file
@@ -0,0 +1,295 @@
|
||||
import java
|
||||
import semmle.code.java.dataflow.internal.DataFlowPrivate
|
||||
import semmle.code.java.dataflow.ExternalFlow
|
||||
import semmle.code.java.dataflow.FlowSummary
|
||||
|
||||
bindingset[this]
|
||||
abstract class CsvRow extends string { }
|
||||
|
||||
Type getParameterType(SummarizedCallableExternal callable, int i) {
|
||||
if i = -1 then result = callable.getDeclaringType() else result = callable.getParameterType(i)
|
||||
}
|
||||
|
||||
string getZero(PrimitiveType t) {
|
||||
t.hasName("float") and result = "0.0f"
|
||||
or
|
||||
t.hasName("double") and result = "0.0"
|
||||
or
|
||||
t.hasName("int") and result = "0"
|
||||
or
|
||||
t.hasName("boolean") and result = "false"
|
||||
or
|
||||
t.hasName("short") and result = "(short)0"
|
||||
or
|
||||
t.hasName("byte") and result = "(byte)0"
|
||||
or
|
||||
t.hasName("char") and result = "'a'"
|
||||
or
|
||||
t.hasName("long") and result = "0L"
|
||||
}
|
||||
|
||||
string getFiller(Type t) {
|
||||
t instanceof RefType and result = "null"
|
||||
or
|
||||
result = getZero(t)
|
||||
}
|
||||
|
||||
Content getContent(SummaryComponent component) { component = SummaryComponent::content(result) }
|
||||
|
||||
string getFieldToken(FieldContent fc) {
|
||||
result =
|
||||
fc.getField().getDeclaringType().getSourceDeclaration().getName() + "_" +
|
||||
fc.getField().getName()
|
||||
}
|
||||
|
||||
string contentToken(Content c) {
|
||||
c instanceof ArrayContent and result = "ArrayElement"
|
||||
or
|
||||
c instanceof CollectionContent and result = "Element"
|
||||
or
|
||||
c instanceof MapKeyContent and result = "MapKey"
|
||||
or
|
||||
c instanceof MapValueContent and result = "MapValue"
|
||||
or
|
||||
result = getFieldToken(c)
|
||||
}
|
||||
|
||||
RefType getRootType(RefType t) {
|
||||
if t instanceof NestedType
|
||||
then result = getRootType(t.(NestedType).getEnclosingType())
|
||||
else result = t
|
||||
}
|
||||
|
||||
Type getRootSourceDeclaration(Type t) {
|
||||
if t instanceof RefType then result = getRootType(t).getSourceDeclaration() else result = t
|
||||
}
|
||||
|
||||
newtype TRowTestSnippet =
|
||||
MkSnippet(
|
||||
CsvRow row, SummarizedCallableExternal callable, SummaryComponentStack input,
|
||||
SummaryComponentStack output, boolean preservesValue
|
||||
) {
|
||||
callable.propagatesFlowForRow(input, output, preservesValue, row)
|
||||
}
|
||||
|
||||
class RowTestSnippet extends TRowTestSnippet {
|
||||
string row;
|
||||
SummarizedCallableExternal callable;
|
||||
SummaryComponentStack input;
|
||||
SummaryComponentStack output;
|
||||
SummaryComponentStack baseInput;
|
||||
SummaryComponentStack baseOutput;
|
||||
boolean preservesValue;
|
||||
|
||||
RowTestSnippet() {
|
||||
this = MkSnippet(row, callable, input, output, preservesValue) and
|
||||
baseInput = input.drop(input.length() - 1) and
|
||||
baseOutput = output.drop(output.length() - 1)
|
||||
}
|
||||
|
||||
string toString() {
|
||||
result =
|
||||
row + " / " + callable + " / " + input + " / " + output + " / " + baseInput + " / " +
|
||||
baseOutput + " / " + preservesValue
|
||||
}
|
||||
|
||||
string getArgument(int i) {
|
||||
(i = -1 or exists(callable.getParameter(i))) and
|
||||
if baseInput = SummaryComponentStack::argument(i)
|
||||
then result = "in"
|
||||
else (
|
||||
if baseOutput = SummaryComponentStack::argument(i)
|
||||
then result = "out"
|
||||
else (
|
||||
if i = -1 then result = "instance" else result = getFiller(getParameterType(callable, i))
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
string makeCall() {
|
||||
// out = in.method(filler);
|
||||
// or
|
||||
// out = filler.method(filler, in, filler);
|
||||
// or
|
||||
// out = Type.method(filler, in, filler);
|
||||
// or
|
||||
// filler.method(filler, in, out, filler);
|
||||
// or
|
||||
// Type.method(filler, in, out, filler);
|
||||
// or
|
||||
// out = new Type(filler, in, filler);
|
||||
// or
|
||||
// new Type(filler, in, out, filler);
|
||||
exists(string storePrefix, string invokePrefix, string args |
|
||||
(
|
||||
if baseOutput = SummaryComponentStack::return()
|
||||
then storePrefix = "out = "
|
||||
else storePrefix = ""
|
||||
) and
|
||||
(
|
||||
if callable instanceof Constructor
|
||||
then invokePrefix = "new "
|
||||
else
|
||||
if callable.(Method).isStatic()
|
||||
then
|
||||
invokePrefix =
|
||||
getShortNameIfPossible(callable.getDeclaringType().getSourceDeclaration()) + "."
|
||||
else invokePrefix = this.getArgument(-1) + "."
|
||||
) and
|
||||
args = concat(int i | i >= 0 | this.getArgument(i), ", " order by i) and
|
||||
result = storePrefix + invokePrefix + callable.getName() + "(" + args + ")"
|
||||
)
|
||||
}
|
||||
|
||||
string getExpectation() {
|
||||
preservesValue = true and result = "// $hasValueFlow"
|
||||
or
|
||||
preservesValue = false and result = "// $hasTaintFlow"
|
||||
}
|
||||
|
||||
string getInstancePrefix() {
|
||||
if
|
||||
callable instanceof Method and
|
||||
not callable.(Method).isStatic() and
|
||||
baseOutput != SummaryComponentStack::argument(-1) and
|
||||
baseInput != SummaryComponentStack::argument(-1)
|
||||
then
|
||||
// In this case `out` is the instance.
|
||||
result = getShortNameIfPossible(callable.getDeclaringType()) + " instance = null;\n"
|
||||
else result = ""
|
||||
}
|
||||
|
||||
Type getOutputType() {
|
||||
if baseOutput = SummaryComponentStack::return()
|
||||
then result = callable.getReturnType()
|
||||
else
|
||||
exists(int i |
|
||||
baseOutput = SummaryComponentStack::argument(i) and
|
||||
result = getParameterType(callable, i)
|
||||
)
|
||||
}
|
||||
|
||||
Type getInputType() {
|
||||
exists(int i |
|
||||
baseInput = SummaryComponentStack::argument(i) and
|
||||
result = getParameterType(callable, i)
|
||||
)
|
||||
}
|
||||
|
||||
string getInputTypeString() { result = getShortNameIfPossible(this.getInputType()) }
|
||||
|
||||
string getInput(SummaryComponentStack componentStack) {
|
||||
componentStack = input.drop(_) and
|
||||
(
|
||||
if componentStack = baseInput
|
||||
then result = "source()"
|
||||
else
|
||||
result =
|
||||
"newWith" + contentToken(getContent(componentStack.head())) + "(" +
|
||||
this.getInput(componentStack.tail()) + ")"
|
||||
)
|
||||
}
|
||||
|
||||
string getOutput(SummaryComponentStack componentStack) {
|
||||
componentStack = output.drop(_) and
|
||||
(
|
||||
if componentStack = baseOutput
|
||||
then result = "out"
|
||||
else
|
||||
result =
|
||||
"get" + contentToken(getContent(componentStack.head())) + "(" +
|
||||
this.getOutput(componentStack.tail()) + ")"
|
||||
)
|
||||
}
|
||||
|
||||
string getASupportMethod() {
|
||||
result =
|
||||
"Object newWith" + contentToken(getContent(input.drop(_).head())) +
|
||||
"(Object element) { return null; }" or
|
||||
result =
|
||||
"Object get" + contentToken(getContent(output.drop(_).head())) +
|
||||
"(Object container) { return null; }"
|
||||
}
|
||||
|
||||
string getASupportMethodModel() {
|
||||
exists(SummaryComponent c, string contentSsvDescription |
|
||||
c = input.drop(_).head() and c = interpretComponent(contentSsvDescription)
|
||||
|
|
||||
result =
|
||||
"generatedtest;Test;false;newWith" + contentToken(getContent(c)) + ";;;Argument[0];" +
|
||||
contentSsvDescription + " of ReturnValue;value"
|
||||
)
|
||||
or
|
||||
exists(SummaryComponent c, string contentSsvDescription |
|
||||
c = output.drop(_).head() and c = interpretComponent(contentSsvDescription)
|
||||
|
|
||||
result =
|
||||
"generatedtest;Test;false;get" + contentToken(getContent(c)) + ";;;" + contentSsvDescription
|
||||
+ " of Argument[0];ReturnValue;value"
|
||||
)
|
||||
}
|
||||
|
||||
Type getADesiredImport() {
|
||||
result =
|
||||
getRootSourceDeclaration([
|
||||
this.getOutputType(), this.getInputType(), callable.getDeclaringType()
|
||||
])
|
||||
}
|
||||
|
||||
string getATestSnippetForRow(string row_) {
|
||||
row_ = row and
|
||||
result =
|
||||
"\t\t{\n\t\t\t// \"" + row + "\"\n\t\t\t" + getShortNameIfPossible(this.getOutputType()) +
|
||||
" out = null;\n\t\t\t" + this.getInputTypeString() + " in = (" + this.getInputTypeString() +
|
||||
")" + this.getInput(input) + ";\n\t\t\t" + this.getInstancePrefix() + this.makeCall() +
|
||||
";\n\t\t\t" + "sink(" + this.getOutput(output) + "); " + this.getExpectation() + "\n\t\t}\n"
|
||||
}
|
||||
}
|
||||
|
||||
predicate isImportable(Type t) {
|
||||
t = any(RowTestSnippet r).getADesiredImport() and
|
||||
t =
|
||||
unique(Type sharesBaseName |
|
||||
sharesBaseName = any(RowTestSnippet r).getADesiredImport() and
|
||||
sharesBaseName.getName() = t.getName()
|
||||
|
|
||||
sharesBaseName
|
||||
)
|
||||
}
|
||||
|
||||
string getShortNameIfPossible(Type t) {
|
||||
getRootSourceDeclaration(t) = any(RowTestSnippet r).getADesiredImport() and
|
||||
if t instanceof RefType and not isImportable(getRootSourceDeclaration(t))
|
||||
then result = t.(RefType).getPackage().getName() + "." + t.getName()
|
||||
else result = t.getName()
|
||||
}
|
||||
|
||||
string getAnImportStatement() {
|
||||
exists(RefType t |
|
||||
t = any(RowTestSnippet r).getADesiredImport() and
|
||||
isImportable(t) and
|
||||
t.getPackage().getName() != "java.lang"
|
||||
|
|
||||
result = "import " + t.getPackage().getName() + "." + t.getName() + ";"
|
||||
)
|
||||
}
|
||||
|
||||
string getASupportMethod() {
|
||||
result = "Object source() { return null; }" or
|
||||
result = "void sink(Object o) { }" or
|
||||
result = any(RowTestSnippet r).getASupportMethod()
|
||||
}
|
||||
|
||||
query string getASupportMethodModel() { result = any(RowTestSnippet r).getASupportMethodModel() }
|
||||
|
||||
query string getTestCase() {
|
||||
result =
|
||||
"package generatedtest;\n\n" + concat(getAnImportStatement() + "\n") +
|
||||
"\n//Test case generated by GenerateFlowTestCase.ql\npublic class Test {\n\n" +
|
||||
concat("\t" + getASupportMethod() + "\n") + "\n\tpublic void test() {\n\n" +
|
||||
concat(string row, string snippet |
|
||||
snippet = any(RowTestSnippet r).getATestSnippetForRow(row)
|
||||
|
|
||||
snippet order by row
|
||||
) + "\n\t}\n\n}"
|
||||
}
|
||||
41
java/ql/src/utils/testFooter.qlfrag
Normal file
41
java/ql/src/utils/testFooter.qlfrag
Normal file
@@ -0,0 +1,41 @@
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
class ValueFlowConf extends DataFlow::Configuration {
|
||||
ValueFlowConf() { this = "qltest:valueFlowConf" }
|
||||
|
||||
override predicate isSource(DataFlow::Node n) { n.asExpr().(MethodAccess).getMethod().hasName("source") }
|
||||
|
||||
override predicate isSink(DataFlow::Node n) { n.asExpr().(Argument).getCall().getCallee().hasName("sink") }
|
||||
}
|
||||
|
||||
class TaintFlowConf extends TaintTracking::Configuration {
|
||||
TaintFlowConf() { this = "qltest:taintFlowConf" }
|
||||
|
||||
override predicate isSource(DataFlow::Node n) { n.asExpr().(MethodAccess).getMethod().hasName("source") }
|
||||
|
||||
override predicate isSink(DataFlow::Node n) { n.asExpr().(Argument).getCall().getCallee().hasName("sink") }
|
||||
}
|
||||
|
||||
class HasFlowTest extends InlineExpectationsTest {
|
||||
HasFlowTest() { this = "HasFlowTest" }
|
||||
|
||||
override string getARelevantTag() { result = ["hasValueFlow", "hasTaintFlow"] }
|
||||
|
||||
override predicate hasActualResult(Location location, string element, string tag, string value) {
|
||||
tag = "hasValueFlow" and
|
||||
exists(DataFlow::Node src, DataFlow::Node sink, ValueFlowConf conf | conf.hasFlow(src, sink) |
|
||||
sink.getLocation() = location and
|
||||
element = sink.toString() and
|
||||
value = ""
|
||||
)
|
||||
or
|
||||
tag = "hasTaintFlow" and
|
||||
exists(DataFlow::Node src, DataFlow::Node sink, TaintFlowConf conf | conf.hasFlow(src, sink) |
|
||||
sink.getLocation() = location and
|
||||
element = sink.toString() and
|
||||
value = ""
|
||||
)
|
||||
}
|
||||
}
|
||||
11
java/ql/src/utils/testHeader.qlfrag
Normal file
11
java/ql/src/utils/testHeader.qlfrag
Normal file
@@ -0,0 +1,11 @@
|
||||
import java
|
||||
import semmle.code.java.dataflow.DataFlow
|
||||
import semmle.code.java.dataflow.ExternalFlow
|
||||
import semmle.code.java.dataflow.TaintTracking
|
||||
import TestUtilities.InlineExpectationsTest
|
||||
|
||||
class SummaryModelTest extends SummaryModelCsv {
|
||||
override predicate row(string row) {
|
||||
row =
|
||||
[
|
||||
//"package;type;overrides;name;signature;ext;inputspec;outputspec;kind",
|
||||
Reference in New Issue
Block a user