From a9b232bff4dd1ede8da828a82b1af759b4c8311c Mon Sep 17 00:00:00 2001 From: Rasmus Lerchedahl Petersen Date: Wed, 21 Dec 2022 20:22:00 +0100 Subject: [PATCH] csharp: use shared inline tests - remove from identical-files --- config/identical-files.json | 1 - .../TestUtilities/InlineExpectationsTest.qll | 400 +----------------- .../InlineExpectationsTestPrivate.qll | 9 - 3 files changed, 11 insertions(+), 399 deletions(-) delete mode 100644 csharp/ql/test/TestUtilities/InlineExpectationsTestPrivate.qll diff --git a/config/identical-files.json b/config/identical-files.json index 836db7b665b..5a0e72bc6d9 100644 --- a/config/identical-files.json +++ b/config/identical-files.json @@ -403,7 +403,6 @@ "csharp/ql/lib/semmle/code/csharp/dataflow/internal/rangeanalysis/ControlFlowReachability.qll" ], "Inline Test Expectations": [ - "csharp/ql/test/TestUtilities/InlineExpectationsTest.qll", "java/ql/test/TestUtilities/InlineExpectationsTest.qll", "ruby/ql/test/TestUtilities/InlineExpectationsTest.qll", "ql/ql/test/TestUtilities/InlineExpectationsTest.qll", diff --git a/csharp/ql/test/TestUtilities/InlineExpectationsTest.qll b/csharp/ql/test/TestUtilities/InlineExpectationsTest.qll index 4a22759c32d..19e6ff17ca0 100644 --- a/csharp/ql/test/TestUtilities/InlineExpectationsTest.qll +++ b/csharp/ql/test/TestUtilities/InlineExpectationsTest.qll @@ -1,399 +1,21 @@ /** - * Provides a library for writing QL tests whose success or failure is based on expected results - * embedded in the test source code as comments, rather than the contents of an `.expected` file - * (in that the `.expected` file should always be empty). - * - * To add this framework to a new language: - * - Add a file `InlineExpectationsTestPrivate.qll` that defines a `ExpectationComment` class. This class - * must support a `getContents` method that returns the contents of the given comment, _excluding_ - * the comment indicator itself. It should also define `toString` and `getLocation` as usual. - * - * To create a new inline expectations test: - * - Declare a class that extends `InlineExpectationsTest`. In the characteristic predicate of the - * new class, bind `this` to a unique string (usually the name of the test). - * - Override the `hasActualResult()` predicate to produce the actual results of the query. For each - * result, specify a `Location`, a text description of the element for which the result was - * reported, a short string to serve as the tag to identify expected results for this test, and the - * expected value of the result. - * - Override `getARelevantTag()` to return the set of tags that can be produced by - * `hasActualResult()`. Often this is just a single tag. - * - * Example: - * ```ql - * class ConstantValueTest extends InlineExpectationsTest { - * ConstantValueTest() { this = "ConstantValueTest" } - * - * override string getARelevantTag() { - * // We only use one tag for this test. - * result = "const" - * } - * - * override predicate hasActualResult( - * Location location, string element, string tag, string value - * ) { - * exists(Expr e | - * tag = "const" and // The tag for this test. - * value = e.getValue() and // The expected value. Will only hold for constant expressions. - * location = e.getLocation() and // The location of the result to be reported. - * element = e.toString() // The display text for the result. - * ) - * } - * } - * ``` - * - * There is no need to write a `select` clause or query predicate. All of the differences between - * expected results and actual results will be reported in the `failures()` query predicate. - * - * To annotate the test source code with an expected result, place a comment starting with a `$` on the - * same line as the expected result, with text of the following format as the body of the comment: - * - * `tag=expected-value` - * - * Where `tag` is the value of the `tag` parameter from `hasActualResult()`, and `expected-value` is - * the value of the `value` parameter from `hasActualResult()`. The `=expected-value` portion may be - * omitted, in which case `expected-value` is treated as the empty string. Multiple expectations may - * be placed in the same comment. Any actual result that - * appears on a line that does not contain a matching expected result comment will be reported with - * a message of the form "Unexpected result: tag=value". Any expected result comment for which there - * is no matching actual result will be reported with a message of the form - * "Missing result: tag=expected-value". - * - * Example: - * ```cpp - * int i = x + 5; // $ const=5 - * int j = y + (7 - 3) // $ const=7 const=3 const=4 // The result of the subtraction is a constant. - * ``` - * - * For tests that contain known missing and spurious results, it is possible to further - * annotate that a particular expected result is known to be spurious, or that a particular - * missing result is known to be missing: - * - * `$ SPURIOUS: tag=expected-value` // Spurious result - * `$ MISSING: tag=expected-value` // Missing result - * - * A spurious expectation is treated as any other expected result, except that if there is no - * matching actual result, the message will be of the form "Fixed spurious result: tag=value". A - * missing expectation is treated as if there were no expected result, except that if a - * matching expected result is found, the message will be of the form - * "Fixed missing result: tag=value". - * - * A single line can contain all the expected, spurious and missing results of that line. For instance: - * `$ tag1=value1 SPURIOUS: tag2=value2 MISSING: tag3=value3`. - * - * If the same result value is expected for two or more tags on the same line, there is a shorthand - * notation available: - * - * `tag1,tag2=expected-value` - * - * is equivalent to: - * - * `tag1=expected-value tag2=expected-value` + * Inline expectation tests for CSharp. + * See `shared/util/codeql/util/test/InlineExpectationsTest.qll` */ -private import InlineExpectationsTestPrivate - -/** - * The base class for tests with inline expectations. The test extends this class to provide the actual - * results of the query, which are then compared with the expected results in comments to produce a - * list of failure messages that point out where the actual results differ from the expected - * results. - */ -abstract class InlineExpectationsTest extends string { - bindingset[this] - InlineExpectationsTest() { any() } +private import csharp as CS +private import codeql.util.test.InlineExpectationsTest +private module Impl implements InlineExpectationsTestSig { /** - * Returns all tags that can be generated by this test. Most tests will only ever produce a single - * tag. Any expected result comments for a tag that is not returned by the `getARelevantTag()` - * predicate for an active test will be ignored. This makes it possible to write multiple tests in - * different `.ql` files that all query the same source code. + * A class representing line comments in C# used by the InlineExpectations core code */ - abstract string getARelevantTag(); - - /** - * Returns the actual results of the query that is being tested. Each result consist of the - * following values: - * - `location` - The source code location of the result. Any expected result comment must appear - * on the start line of this location. - * - `element` - Display text for the element on which the result is reported. - * - `tag` - The tag that marks this result as coming from this test. This must be one of the tags - * returned by `getARelevantTag()`. - * - `value` - The value of the result, which will be matched against the value associated with - * `tag` in any expected result comment on that line. - */ - abstract predicate hasActualResult(Location location, string element, string tag, string value); - - /** - * Holds if there is an optional result on the specified location. - * - * This is similar to `hasActualResult`, but returns results that do not require a matching annotation. - * A failure will still arise if there is an annotation that does not match any results, but not vice versa. - * Override this predicate to specify optional results. - */ - predicate hasOptionalResult(Location location, string element, string tag, string value) { - none() + class ExpectationComment extends CS::SinglelineComment { + /** Gets the contents of the given comment, _without_ the preceding comment marker (`//`). */ + string getContents() { result = this.getText() } } - final predicate hasFailureMessage(FailureLocatable element, string message) { - exists(ActualResult actualResult | - actualResult.getTest() = this and - actualResult.getTag() = this.getARelevantTag() and - element = actualResult and - ( - exists(FalseNegativeExpectation falseNegative | - falseNegative.matchesActualResult(actualResult) and - message = "Fixed missing result:" + falseNegative.getExpectationText() - ) - or - not exists(ValidExpectation expectation | expectation.matchesActualResult(actualResult)) and - message = "Unexpected result: " + actualResult.getExpectationText() and - not actualResult.isOptional() - ) - ) - or - exists(ActualResult actualResult | - actualResult.getTest() = this and - not actualResult.getTag() = this.getARelevantTag() and - element = actualResult and - message = - "Tag mismatch: Actual result with tag '" + actualResult.getTag() + - "' that is not part of getARelevantTag()" - ) - or - exists(ValidExpectation expectation | - not exists(ActualResult actualResult | expectation.matchesActualResult(actualResult)) and - expectation.getTag() = this.getARelevantTag() and - element = expectation and - ( - expectation instanceof GoodExpectation and - message = "Missing result:" + expectation.getExpectationText() - or - expectation instanceof FalsePositiveExpectation and - message = "Fixed spurious result:" + expectation.getExpectationText() - ) - ) - or - exists(InvalidExpectation expectation | - element = expectation and - message = "Invalid expectation syntax: " + expectation.getExpectation() - ) - } + class Location = CS::Location; } -/** - * RegEx pattern to match a comment containing one or more expected results. The comment must have - * `$` as its first non-whitespace character. Any subsequent character - * is treated as part of the expected results, except that the comment may contain a `//` sequence - * to treat the remainder of the line as a regular (non-interpreted) comment. - */ -private string expectationCommentPattern() { result = "\\s*\\$((?:[^/]|/[^/])*)(?://.*)?" } - -/** - * The possible columns in an expectation comment. The `TDefaultColumn` branch represents the first - * column in a comment. This column is not precedeeded by a name. `TNamedColumn(name)` represents a - * column containing expected results preceded by the string `name:`. - */ -private newtype TColumn = - TDefaultColumn() or - TNamedColumn(string name) { name = ["MISSING", "SPURIOUS"] } - -bindingset[start, content] -private int getEndOfColumnPosition(int start, string content) { - result = - min(string name, int cand | - exists(TNamedColumn(name)) and - cand = content.indexOf(name + ":") and - cand >= start - | - cand - ) - or - not exists(string name | - exists(TNamedColumn(name)) and - content.indexOf(name + ":") >= start - ) and - result = content.length() -} - -private predicate getAnExpectation( - ExpectationComment comment, TColumn column, string expectation, string tags, string value -) { - exists(string content | - content = comment.getContents().regexpCapture(expectationCommentPattern(), 1) and - ( - column = TDefaultColumn() and - exists(int end | - end = getEndOfColumnPosition(0, content) and - expectation = content.prefix(end).regexpFind(expectationPattern(), _, _).trim() - ) - or - exists(string name, int start, int end | - column = TNamedColumn(name) and - start = content.indexOf(name + ":") + name.length() + 1 and - end = getEndOfColumnPosition(start, content) and - expectation = content.substring(start, end).regexpFind(expectationPattern(), _, _).trim() - ) - ) - ) and - tags = expectation.regexpCapture(expectationPattern(), 1) and - if exists(expectation.regexpCapture(expectationPattern(), 2)) - then value = expectation.regexpCapture(expectationPattern(), 2) - else value = "" -} - -private string getColumnString(TColumn column) { - column = TDefaultColumn() and result = "" - or - column = TNamedColumn(result) -} - -/** - * RegEx pattern to match a single expected result, not including the leading `$`. It consists of one or - * more comma-separated tags optionally followed by `=` and the expected value. - * - * Tags must be only letters, digits, `-` and `_` (note that the first character - * must not be a digit), but can contain anything enclosed in a single set of - * square brackets. - * - * Examples: - * - `tag` - * - `tag=value` - * - `tag,tag2=value` - * - `tag[foo bar]=value` - * - * Not allowed: - * - `tag[[[foo bar]` - */ -private string expectationPattern() { - exists(string tag, string tags, string value | - tag = "[A-Za-z-_](?:[A-Za-z-_0-9]|\\[[^\\]\\]]*\\])*" and - tags = "((?:" + tag + ")(?:\\s*,\\s*" + tag + ")*)" and - // In Python, we allow both `"` and `'` for strings, as well as the prefixes `bru`. - // For example, `b"foo"`. - value = "((?:[bru]*\"[^\"]*\"|[bru]*'[^']*'|\\S+)*)" and - result = tags + "(?:=" + value + ")?" - ) -} - -private newtype TFailureLocatable = - TActualResult( - InlineExpectationsTest test, Location location, string element, string tag, string value, - boolean optional - ) { - test.hasActualResult(location, element, tag, value) and - optional = false - or - test.hasOptionalResult(location, element, tag, value) and optional = true - } or - TValidExpectation(ExpectationComment comment, string tag, string value, string knownFailure) { - exists(TColumn column, string tags | - getAnExpectation(comment, column, _, tags, value) and - tag = tags.splitAt(",") and - knownFailure = getColumnString(column) - ) - } or - TInvalidExpectation(ExpectationComment comment, string expectation) { - getAnExpectation(comment, _, expectation, _, _) and - not expectation.regexpMatch(expectationPattern()) - } - -class FailureLocatable extends TFailureLocatable { - string toString() { none() } - - Location getLocation() { none() } - - final string getExpectationText() { result = getTag() + "=" + getValue() } - - string getTag() { none() } - - string getValue() { none() } -} - -class ActualResult extends FailureLocatable, TActualResult { - InlineExpectationsTest test; - Location location; - string element; - string tag; - string value; - boolean optional; - - ActualResult() { this = TActualResult(test, location, element, tag, value, optional) } - - override string toString() { result = element } - - override Location getLocation() { result = location } - - InlineExpectationsTest getTest() { result = test } - - override string getTag() { result = tag } - - override string getValue() { result = value } - - predicate isOptional() { optional = true } -} - -abstract private class Expectation extends FailureLocatable { - ExpectationComment comment; - - override string toString() { result = comment.toString() } - - override Location getLocation() { result = comment.getLocation() } -} - -private predicate onSameLine(ValidExpectation a, ActualResult b) { - exists(string fname, int line, Location la, Location lb | - // Join order intent: - // Take the locations of ActualResults, - // join with locations in the same file / on the same line, - // then match those against ValidExpectations. - la = a.getLocation() and - pragma[only_bind_into](lb) = b.getLocation() and - pragma[only_bind_into](la).hasLocationInfo(fname, line, _, _, _) and - lb.hasLocationInfo(fname, line, _, _, _) - ) -} - -private class ValidExpectation extends Expectation, TValidExpectation { - string tag; - string value; - string knownFailure; - - ValidExpectation() { this = TValidExpectation(comment, tag, value, knownFailure) } - - override string getTag() { result = tag } - - override string getValue() { result = value } - - string getKnownFailure() { result = knownFailure } - - predicate matchesActualResult(ActualResult actualResult) { - onSameLine(pragma[only_bind_into](this), actualResult) and - getTag() = actualResult.getTag() and - getValue() = actualResult.getValue() - } -} - -/* Note: These next three classes correspond to all the possible values of type `TColumn`. */ -class GoodExpectation extends ValidExpectation { - GoodExpectation() { getKnownFailure() = "" } -} - -class FalsePositiveExpectation extends ValidExpectation { - FalsePositiveExpectation() { getKnownFailure() = "SPURIOUS" } -} - -class FalseNegativeExpectation extends ValidExpectation { - FalseNegativeExpectation() { getKnownFailure() = "MISSING" } -} - -class InvalidExpectation extends Expectation, TInvalidExpectation { - string expectation; - - InvalidExpectation() { this = TInvalidExpectation(comment, expectation) } - - string getExpectation() { result = expectation } -} - -query predicate failures(FailureLocatable element, string message) { - exists(InlineExpectationsTest test | test.hasFailureMessage(element, message)) -} +import Make diff --git a/csharp/ql/test/TestUtilities/InlineExpectationsTestPrivate.qll b/csharp/ql/test/TestUtilities/InlineExpectationsTestPrivate.qll deleted file mode 100644 index 4f3c3f6bef5..00000000000 --- a/csharp/ql/test/TestUtilities/InlineExpectationsTestPrivate.qll +++ /dev/null @@ -1,9 +0,0 @@ -import csharp - -/** - * A class representing line comments in C# used by the InlineExpectations core code - */ -class ExpectationComment extends SinglelineComment { - /** Gets the contents of the given comment, _without_ the preceding comment marker (`//`). */ - string getContents() { result = this.getText() } -}