hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2026-04-22 07:15:15 +02:00
Code Issues Packages Projects Releases Wiki Activity

Shared: Post-processing query for inline test expectations

Browse Source
This commit is contained in:
Tom Hvitved
2024-09-20 13:58:49 +02:00
parent b111194fbc
commit e7a3e6bfed
1 changed files with 309 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

318
shared/util/codeql/util/test/InlineExpectationsTest.qll
View File

@@ -134,8 +134,31 @@ module Make<InlineExpectationsTestSig Impl> {
* predicate for an active test will be ignored. This makes it possible to write multiple tests in
* different `.ql` files that all query the same source code.
*/
bindingset[result]
string getARelevantTag();
/**
* Holds if expected tag `expectedTag` matches actual tag `actualTag`.
*
* This is normally defined as `expectedTag = actualTag`.
*/
bindingset[expectedTag, actualTag]
default predicate tagMatches(string expectedTag, string actualTag) { expectedTag = actualTag }
/** Holds if expectations marked with `expectedTag` are optional. */
bindingset[expectedTag]
default predicate tagIsOptional(string expectedTag) { none() }
/**
* Holds if expected value `expectedValue` matches actual value `actualValue`.
*
* This is normally defined as `expectedValue = actualValue`.
*/
bindingset[expectedValue, actualValue]
default predicate valueMatches(string expectedValue, string actualValue) {
expectedValue = actualValue
}
/**
* Returns the actual results of the query that is being tested. Each result consist of the
* following values:
@@ -200,13 +223,13 @@ module Make<InlineExpectationsTestSig Impl> {
not exists(ActualTestResult actualResult | expectation.matchesActualResult(actualResult)) and
expectation.getTag() = TestImpl::getARelevantTag() and
element = expectation and
(
expectation instanceof GoodTestExpectation and
message = "Missing result: " + expectation.getExpectationText()
or
expectation instanceof FalsePositiveTestExpectation and
message = "Fixed spurious result: " + expectation.getExpectationText()
)
not expectation.isOptional()
|
expectation instanceof GoodTestExpectation and
message = "Missing result: " + expectation.getExpectationText()
or
expectation instanceof FalsePositiveTestExpectation and
message = "Fixed spurious result: " + expectation.getExpectationText()
)
or
exists(InvalidTestExpectation expectation |
@@ -311,9 +334,11 @@ module Make<InlineExpectationsTestSig Impl> {
predicate matchesActualResult(ActualTestResult actualResult) {
onSameLine(pragma[only_bind_into](this), actualResult) and
this.getTag() = actualResult.getTag() and
this.getValue() = actualResult.getValue()
TestImpl::tagMatches(this.getTag(), actualResult.getTag()) and
TestImpl::valueMatches(this.getValue(), actualResult.getValue())
}
predicate isOptional() { TestImpl::tagIsOptional(tag) }
}
// Note: These next three classes correspond to all the possible values of type `TColumn`.
@@ -337,6 +362,18 @@ module Make<InlineExpectationsTestSig Impl> {
string getExpectation() { result = expectation }
}
/**
* Gets a test expectation that matches the actual result at the given location.
*/
ValidTestExpectation getAMatchingExpectation(
Impl::Location location, string element, string tag, string val, boolean optional
) {
exists(ActualTestResult actualResult |
result.matchesActualResult(actualResult) and
actualResult = TActualResult(location, element, tag, val, optional)
)
}
query predicate testFailures(FailureLocatable element, string message) {
hasFailureMessage(element, message)
}
@@ -385,6 +422,7 @@ module Make<InlineExpectationsTestSig Impl> {
* ```
*/
module MergeTests<TestSig TestImpl1, TestSig TestImpl2> implements TestSig {
bindingset[result]
string getARelevantTag() {
result = TestImpl1::getARelevantTag() or result = TestImpl2::getARelevantTag()
}
@@ -408,6 +446,7 @@ module Make<InlineExpectationsTestSig Impl> {
module MergeTests3<TestSig TestImpl1, TestSig TestImpl2, TestSig TestImpl3> implements TestSig {
private module M = MergeTests<MergeTests<TestImpl1, TestImpl2>, TestImpl3>;
bindingset[result]
string getARelevantTag() { result = M::getARelevantTag() }
predicate hasActualResult(Impl::Location location, string element, string tag, string value) {
@@ -427,6 +466,7 @@ module Make<InlineExpectationsTestSig Impl> {
{
private module M = MergeTests<MergeTests3<TestImpl1, TestImpl2, TestImpl3>, TestImpl4>;
bindingset[result]
string getARelevantTag() { result = M::getARelevantTag() }
predicate hasActualResult(Impl::Location location, string element, string tag, string value) {
@@ -448,6 +488,7 @@ module Make<InlineExpectationsTestSig Impl> {
private module M =
MergeTests<MergeTests4<TestImpl1, TestImpl2, TestImpl3, TestImpl4>, TestImpl5>;
bindingset[result]
string getARelevantTag() { result = M::getARelevantTag() }
predicate hasActualResult(Impl::Location location, string element, string tag, string value) {
@@ -590,3 +631,262 @@ private string expectationPattern() {
result = tags + "(?:=" + value + ")?"
)
}
/**
* Provides logic for creating a `@kind test-postprocess` query that checks
* inline test expectations using `$ Alert` markers.
*
* The postprocessing query works for queries of kind `problem` and `path-problem`,
* and each query result must have a matching `$ Alert` comment. It is possible to
* augment the comment with a query ID, in order to support cases where multiple
* `.qlref` tests share the same test code:
*
* ```rust
* var x = ""; // $ Alert[rust/unused-value]
* return;
* foo(); // $ Alert[rust/unreachable-code]
* ```
*
* In the example above, the `$ Alert[rust/unused-value]` commment is only taken
* into account in the test for the query with ID `rust/unused-value`, and vice
* versa for the `$ Alert[rust/unreachable-code]` comment.
*
* For `path-problem` queries, each source and sink must additionally be annotated
* (`$ Source` and `$ Sink`, respectively), except when their location coincides
* with the location of the alert itself, in which case only `$ Alert` is needed.
*
* Example:
*
* ```csharp
* var queryParam = Request.QueryString["param"]; // $ Source
* Write(Html.Raw(queryParam)); // $ Alert
* ```
*
* Morover, it is possible to tag sources with a unique identifier:
*
* ```csharp
* var queryParam = Request.QueryString["param"]; // $ Source=source1
* Write(Html.Raw(queryParam)); // $ Alert=source1
* ```
*
* In this case, the source and sink must have the same tag in order
* to be matched.
*/
module TestPostProcessing {
external predicate queryResults(string relation, int row, int column, string data);
external predicate queryRelations(string relation);
external predicate queryMetadata(string key, string value);
private string getQueryId() { queryMetadata("id", result) }
private string getQueryKind() { queryMetadata("kind", result) }
signature module InputSig<InlineExpectationsTestSig Input> {
string getRelativeUrl(Input::Location location);
}
module Make<InlineExpectationsTestSig Input, InputSig<Input> Input2> {
private import InlineExpectationsTest as InlineExpectationsTest
private import InlineExpectationsTest::Make<Input>
/**
* Gets the tag to be used for the path-problem source at result row `row`.
*
* This is either `Source` or `Alert`, depending on whether the location
* of the source matches the location of the alert.
*/
private string getSourceTag(int row) {
getQueryKind() = "path-problem" and
exists(string loc | queryResults("#select", row, 2, loc) |
if queryResults("#select", row, 0, loc) then result = "Alert" else result = "Source"
)
}
/**
* Gets the tag to be used for the path-problem sink at result row `row`.
*
* This is either `Sink` or `Alert`, depending on whether the location
* of the sink matches the location of the alert.
*/
private string getSinkTag(int row) {
getQueryKind() = "path-problem" and
exists(string loc | queryResults("#select", row, 4, loc) |
if queryResults("#select", row, 0, loc) then result = "Alert" else result = "Sink"
)
}
/**
* A configuration for matching `// $ Source=foo` comments against actual
* path-problem sources.
*/
private module PathProblemSourceTestInput implements TestSig {
string getARelevantTag() { result = getSourceTag(_) }
bindingset[expectedValue, actualValue]
predicate valueMatches(string expectedValue, string actualValue) {
exists(expectedValue) and
actualValue = ""
}
additional predicate hasPathProblemSource(
int row, Input::Location location, string element, string tag, string value
) {
getQueryKind() = "path-problem" and
exists(string loc |
queryResults("#select", row, 2, loc) and
queryResults("#select", row, 3, element) and
tag = getSourceTag(row) and
value = "" and
Input2::getRelativeUrl(location) = loc
)
}
predicate hasActualResult(Input::Location location, string element, string tag, string value) {
hasPathProblemSource(_, location, element, tag, value)
}
}
private module PathProblemSourceTest = MakeTest<PathProblemSourceTestInput>;
private module TestInput implements TestSig {
bindingset[result]
string getARelevantTag() { any() }
private string getTagRegex() {
exists(string sourceSinkTags |
getQueryKind() = "problem" and
sourceSinkTags = ""
or
sourceSinkTags = "|" + getSourceTag(_) + "|" + getSinkTag(_)
|
result = "(Alert" + sourceSinkTags + ")(\\[(.*)\\])?"
)
}
bindingset[expectedTag, actualTag]
predicate tagMatches(string expectedTag, string actualTag) {
actualTag = expectedTag.regexpCapture(getTagRegex(), 1) and
(
// expected tag is annotated with a query ID
getQueryId() = expectedTag.regexpCapture(getTagRegex(), 3)
or
// expected tag is not annotated with a query ID
not exists(expectedTag.regexpCapture(getTagRegex(), 3))
)
}
bindingset[expectedTag]
predicate tagIsOptional(string expectedTag) {
// ignore irrelevant tags
not expectedTag.regexpMatch(getTagRegex())
or
// ignore tags annotated with a query ID that does not match the current query ID
exists(string queryId |
queryId = expectedTag.regexpCapture(getTagRegex(), 3) and
queryId != getQueryId()
)
}
bindingset[expectedValue, actualValue]
predicate valueMatches(string expectedValue, string actualValue) {
expectedValue = actualValue
or
actualValue = ""
}
private predicate hasPathProblemSource = PathProblemSourceTestInput::hasPathProblemSource/5;
/**
* Gets the expected sink value for result row `row`. This value must
* match the value at the corresponding path-problem source (if it is
* present).
*/
private string getSinkValue(int row) {
exists(Input::Location location, string element, string tag, string val |
hasPathProblemSource(row, location, element, tag, val) and
result =
PathProblemSourceTest::getAMatchingExpectation(location, element, tag, val, false)
.getValue()
)
}
private predicate hasPathProblemSink(
int row, Input::Location location, string element, string tag, string value
) {
getQueryKind() = "path-problem" and
exists(string loc |
queryResults("#select", row, 4, loc) and
queryResults("#select", row, 5, element) and
tag = getSinkTag(row) and
Input2::getRelativeUrl(location) = loc
|
not exists(getSinkValue(row)) and value = ""
or
value = getSinkValue(row)
)
}
private predicate hasAlert(Input::Location location, string element, string tag, string value) {
getQueryKind() = ["problem", "path-problem"] and
exists(int row, string loc |
queryResults("#select", row, 0, loc) and
queryResults("#select", row, 2, element) and
tag = "Alert" and
value = "" and
Input2::getRelativeUrl(location) = loc and
not hasPathProblemSource(row, location, _, _, _) and
not hasPathProblemSink(row, location, _, _, _)
)
}
predicate hasActualResult(Input::Location location, string element, string tag, string value) {
hasPathProblemSource(_, location, element, tag, value)
or
hasPathProblemSink(_, location, element, tag, value)
or
hasAlert(location, element, tag, value)
}
}
private module Test = MakeTest<TestInput>;
private newtype TTestFailure =
MkTestFailure(Test::FailureLocatable f, string message) { Test::testFailures(f, message) }
private predicate rankedTestFailures(int i, MkTestFailure f) {
f =
rank[i](MkTestFailure f0, Test::FailureLocatable fl, string message, string filename,
int startLine, int startColumn, int endLine, int endColumn |
f0 = MkTestFailure(fl, message) and
fl.getLocation().hasLocationInfo(filename, startLine, startColumn, endLine, endColumn)
|
f0 order by filename, startLine, startColumn, endLine, endColumn, message
)
}
query predicate results(string relation, int row, int column, string data) {
queryResults(relation, row, column, data)
or
exists(MkTestFailure f, Test::FailureLocatable fl, string message |
relation = "testFailures" and
rankedTestFailures(row, f) and
f = MkTestFailure(fl, message)
|
column = 0 and data = Input2::getRelativeUrl(fl.getLocation())
or
column = 1 and data = fl.toString()
or
column = 2 and data = message
)
}
query predicate resultRelations(string relation) {
queryRelations(relation)
or
Test::testFailures(_, _) and
relation = "testFailures"
}
}
}