Use inline expectations for negative-example tests as well.

This commit is contained in:
Max Schaefer
2024-01-16 17:44:05 +00:00
parent 1ebd0747a8
commit 83c567385f
7 changed files with 87 additions and 53 deletions

View File

@@ -397,6 +397,45 @@ predicate isCandidate(
)
}
/**
* Holds if the given `endpoint` is a negative example for the `extensibleType`
* because of the `characteristic`.
*
* The other parameters record various other properties of interest.
*/
predicate isNegativeExample(
Endpoint endpoint, EndpointCharacteristic characteristic, float confidence, string package,
string type, string subtypes, string name, string signature, string input, string output,
string isVarargsArray, string extensibleType
) {
characteristic.appliesToEndpoint(endpoint) and
// the node is known not to be an endpoint of any appropriate type
forall(AutomodelEndpointTypes::EndpointType tp |
tp = CharacteristicsImpl::getAPotentialType(endpoint)
|
characteristic.hasImplications(tp, false, _)
) and
// the lowest confidence across all endpoint types should be at least highConfidence
confidence =
min(float c |
characteristic.hasImplications(CharacteristicsImpl::getAPotentialType(endpoint), false, c)
) and
confidence >= SharedCharacteristics::highConfidence() and
any(ApplicationModeMetadataExtractor meta)
.hasMetadata(endpoint, package, type, subtypes, name, signature, input, output,
isVarargsArray, _, extensibleType) and
// It's valid for a node to be both a potential source/sanitizer and a sink. We don't want to include such nodes
// as negative examples in the prompt, because they're ambiguous and might confuse the model, so we explicitly exclude them here.
not exists(EndpointCharacteristic characteristic2, float confidence2 |
characteristic2 != characteristic
|
characteristic2.appliesToEndpoint(endpoint) and
confidence2 >= SharedCharacteristics::maximalConfidence() and
characteristic2
.hasImplications(CharacteristicsImpl::getAPotentialType(endpoint), true, confidence2)
)
}
/*
* EndpointCharacteristic classes that are specific to Automodel for Java.
*/

View File

@@ -40,36 +40,6 @@ Endpoint getSampleForCharacteristic(EndpointCharacteristic c, int limit) {
)
}
predicate candidate(
Endpoint endpoint, EndpointCharacteristic characteristic, float confidence, string package,
string type, string subtypes, string name, string signature, string input, string output,
string isVarargsArray, string extensibleType
) {
// the node is known not to be an endpoint of any appropriate type
forall(EndpointType tp | tp = CharacteristicsImpl::getAPotentialType(endpoint) |
characteristic.hasImplications(tp, false, _)
) and
// the lowest confidence across all endpoint types should be at least highConfidence
confidence =
min(float c |
characteristic.hasImplications(CharacteristicsImpl::getAPotentialType(endpoint), false, c)
) and
confidence >= SharedCharacteristics::highConfidence() and
any(ApplicationModeMetadataExtractor meta)
.hasMetadata(endpoint, package, type, subtypes, name, signature, input, output,
isVarargsArray, _, extensibleType) and
// It's valid for a node to be both a potential source/sanitizer and a sink. We don't want to include such nodes
// as negative examples in the prompt, because they're ambiguous and might confuse the model, so we explicitly exclude them here.
not exists(EndpointCharacteristic characteristic2, float confidence2 |
characteristic2 != characteristic
|
characteristic2.appliesToEndpoint(endpoint) and
confidence2 >= SharedCharacteristics::maximalConfidence() and
characteristic2
.hasImplications(CharacteristicsImpl::getAPotentialType(endpoint), true, confidence2)
)
}
from
Endpoint endpoint, EndpointCharacteristic characteristic, float confidence, string message,
DollarAtString package, DollarAtString type, DollarAtString subtypes, DollarAtString name,
@@ -77,8 +47,8 @@ from
DollarAtString isVarargsArray, DollarAtString extensibleType
where
endpoint = getSampleForCharacteristic(characteristic, 100) and
candidate(endpoint, characteristic, confidence, package, type, subtypes, name, signature, input,
output, isVarargsArray, extensibleType) and
isNegativeExample(endpoint, characteristic, confidence, package, type, subtypes, name, signature,
input, output, isVarargsArray, extensibleType) and
message = characteristic
select endpoint.asNode(),
message + "\nrelated locations: $@, $@, $@." + "\nmetadata: $@, $@, $@, $@, $@, $@, $@, $@, $@.", //

View File

@@ -1,3 +1,2 @@
| Test.java:49:4:49:5 | f2 | known non-sink\nrelated locations: $@, $@, $@.\nmetadata: $@, $@, $@, $@, $@, $@, $@, $@, $@. | Test.java:48:10:50:3 | compareTo(...) | CallContext | Test.java:49:4:49:5 | f2 | MethodDoc | Test.java:49:4:49:5 | f2 | ClassDoc | file://java.io:1:1:1:1 | java.io | package | file://File:1:1:1:1 | File | type | file://true:1:1:1:1 | true | subtypes | file://compareTo:1:1:1:1 | compareTo | name | file://(File):1:1:1:1 | (File) | signature | file://Argument[0]:1:1:1:1 | Argument[0] | input | file://:1:1:1:1 | | output | file://false:1:1:1:1 | false | isVarargsArray | file://sinkModel:1:1:1:1 | sinkModel | extensibleType |
| Test.java:55:4:55:4 | p | taint step\nrelated locations: $@, $@, $@.\nmetadata: $@, $@, $@, $@, $@, $@, $@, $@, $@. | Test.java:54:3:59:3 | walk(...) | CallContext | Test.java:55:4:55:4 | p | MethodDoc | Test.java:55:4:55:4 | p | ClassDoc | file://java.nio.file:1:1:1:1 | java.nio.file | package | file://Files:1:1:1:1 | Files | type | file://false:1:1:1:1 | false | subtypes | file://walk:1:1:1:1 | walk | name | file://(Path,FileVisitOption[]):1:1:1:1 | (Path,FileVisitOption[]) | signature | file://Argument[0]:1:1:1:1 | Argument[0] | input | file://:1:1:1:1 | | output | file://false:1:1:1:1 | false | isVarargsArray | file://sinkModel:1:1:1:1 | sinkModel | extensibleType |
| Test.java:94:3:96:3 | deleteIfExists(...) | known sanitizer\nrelated locations: $@, $@, $@.\nmetadata: $@, $@, $@, $@, $@, $@, $@, $@, $@. | Test.java:94:3:96:3 | deleteIfExists(...) | CallContext | Test.java:94:3:96:3 | deleteIfExists(...) | MethodDoc | Test.java:94:3:96:3 | deleteIfExists(...) | ClassDoc | file://java.nio.file:1:1:1:1 | java.nio.file | package | file://Files:1:1:1:1 | Files | type | file://false:1:1:1:1 | false | subtypes | file://deleteIfExists:1:1:1:1 | deleteIfExists | name | file://(Path):1:1:1:1 | (Path) | signature | file://:1:1:1:1 | | input | file://ReturnValue:1:1:1:1 | ReturnValue | output | file://false:1:1:1:1 | false | isVarargsArray | file://sourceModel:1:1:1:1 | sourceModel | extensibleType |
testFailures
failures

View File

@@ -0,0 +1,27 @@
import java
import AutomodelApplicationModeCharacteristics
import TestUtilities.InlineExpectationsTest
module NegativeExampleTest implements TestSig {
string getARelevantTag() { result = "negativeExample" }
predicate hasActualResult(Location location, string element, string tag, string value) {
exists(
Endpoint endpoint, string name, string signature, string input,
string output, string extensibleType
|
isNegativeExample(endpoint, _, _, _, _, _, name, signature, input, output, _,
extensibleType)
|
endpoint.asTop().getLocation() = location and
endpoint.toString() = element and
tag = "negativeExample" and
// for source models only the output is relevant, and vice versa for sink models
if extensibleType = "sourceModel"
then value = name + signature + ":" + output
else value = name + signature + ":" + input
)
}
}
import MakeTest<NegativeExampleTest>

View File

@@ -1 +0,0 @@
AutomodelApplicationModeExtractNegativeExamples.ql

View File

@@ -1,7 +1,7 @@
| Test.java:29:4:29:9 | source | path-injection\nrelated locations: $@, $@, $@.\nmetadata: $@, $@, $@, $@, $@, $@, $@, $@, $@. | Test.java:28:3:32:3 | copy(...) | CallContext | Test.java:29:4:29:9 | source | MethodDoc | Test.java:29:4:29:9 | source | ClassDoc | file://java.nio.file:1:1:1:1 | java.nio.file | package | file://Files:1:1:1:1 | Files | type | file://false:1:1:1:1 | false | subtypes | file://copy:1:1:1:1 | copy | name | file://(Path,Path,CopyOption[]):1:1:1:1 | (Path,Path,CopyOption[]) | signature | file://Argument[0]:1:1:1:1 | Argument[0] | input | file://:1:1:1:1 | | output | file://false:1:1:1:1 | false | isVarargsArray | file://sinkModel:1:1:1:1 | sinkModel | extensibleType |
| Test.java:30:4:30:9 | target | path-injection\nrelated locations: $@, $@, $@.\nmetadata: $@, $@, $@, $@, $@, $@, $@, $@, $@. | Test.java:28:3:32:3 | copy(...) | CallContext | Test.java:30:4:30:9 | target | MethodDoc | Test.java:30:4:30:9 | target | ClassDoc | file://java.nio.file:1:1:1:1 | java.nio.file | package | file://Files:1:1:1:1 | Files | type | file://false:1:1:1:1 | false | subtypes | file://copy:1:1:1:1 | copy | name | file://(Path,Path,CopyOption[]):1:1:1:1 | (Path,Path,CopyOption[]) | signature | file://Argument[1]:1:1:1:1 | Argument[1] | input | file://:1:1:1:1 | | output | file://false:1:1:1:1 | false | isVarargsArray | file://sinkModel:1:1:1:1 | sinkModel | extensibleType |
| Test.java:37:4:37:11 | openPath | path-injection\nrelated locations: $@, $@, $@.\nmetadata: $@, $@, $@, $@, $@, $@, $@, $@, $@. | Test.java:36:10:38:3 | newInputStream(...) | CallContext | Test.java:37:4:37:11 | openPath | MethodDoc | Test.java:37:4:37:11 | openPath | ClassDoc | file://java.nio.file:1:1:1:1 | java.nio.file | package | file://Files:1:1:1:1 | Files | type | file://false:1:1:1:1 | false | subtypes | file://newInputStream:1:1:1:1 | newInputStream | name | file://(Path,OpenOption[]):1:1:1:1 | (Path,OpenOption[]) | signature | file://Argument[0]:1:1:1:1 | Argument[0] | input | file://:1:1:1:1 | | output | file://false:1:1:1:1 | false | isVarargsArray | file://sinkModel:1:1:1:1 | sinkModel | extensibleType |
| Test.java:63:3:63:20 | getInputStream(...) | remote\nrelated locations: $@, $@, $@.\nmetadata: $@, $@, $@, $@, $@, $@, $@, $@, $@. | Test.java:63:3:63:20 | getInputStream(...) | CallContext | Test.java:63:3:63:20 | getInputStream(...) | MethodDoc | Test.java:63:3:63:20 | getInputStream(...) | ClassDoc | file://java.net:1:1:1:1 | java.net | package | file://URLConnection:1:1:1:1 | URLConnection | type | file://true:1:1:1:1 | true | subtypes | file://getInputStream:1:1:1:1 | getInputStream | name | file://():1:1:1:1 | () | signature | file://:1:1:1:1 | | input | file://ReturnValue:1:1:1:1 | ReturnValue | output | file://false:1:1:1:1 | false | isVarargsArray | file://sourceModel:1:1:1:1 | sourceModel | extensibleType |
| Test.java:87:28:87:28 | p | path-injection\nrelated locations: $@, $@, $@.\nmetadata: $@, $@, $@, $@, $@, $@, $@, $@, $@. | Test.java:87:4:87:29 | createDirectories(...) | CallContext | Test.java:87:28:87:28 | p | MethodDoc | Test.java:87:28:87:28 | p | ClassDoc | file://java.nio.file:1:1:1:1 | java.nio.file | package | file://Files:1:1:1:1 | Files | type | file://false:1:1:1:1 | false | subtypes | file://createDirectories:1:1:1:1 | createDirectories | name | file://(Path,FileAttribute[]):1:1:1:1 | (Path,FileAttribute[]) | signature | file://Argument[0]:1:1:1:1 | Argument[0] | input | file://:1:1:1:1 | | output | file://false:1:1:1:1 | false | isVarargsArray | file://sinkModel:1:1:1:1 | sinkModel | extensibleType |
| Test.java:91:4:91:4 | p | path-injection\nrelated locations: $@, $@, $@.\nmetadata: $@, $@, $@, $@, $@, $@, $@, $@, $@. | Test.java:90:3:92:3 | delete(...) | CallContext | Test.java:91:4:91:4 | p | MethodDoc | Test.java:91:4:91:4 | p | ClassDoc | file://java.nio.file:1:1:1:1 | java.nio.file | package | file://Files:1:1:1:1 | Files | type | file://false:1:1:1:1 | false | subtypes | file://delete:1:1:1:1 | delete | name | file://(Path):1:1:1:1 | (Path) | signature | file://Argument[0]:1:1:1:1 | Argument[0] | input | file://:1:1:1:1 | | output | file://false:1:1:1:1 | false | isVarargsArray | file://sinkModel:1:1:1:1 | sinkModel | extensibleType |
| Test.java:95:4:95:4 | p | path-injection\nrelated locations: $@, $@, $@.\nmetadata: $@, $@, $@, $@, $@, $@, $@, $@, $@. | Test.java:94:3:96:3 | deleteIfExists(...) | CallContext | Test.java:95:4:95:4 | p | MethodDoc | Test.java:95:4:95:4 | p | ClassDoc | file://java.nio.file:1:1:1:1 | java.nio.file | package | file://Files:1:1:1:1 | Files | type | file://false:1:1:1:1 | false | subtypes | file://deleteIfExists:1:1:1:1 | deleteIfExists | name | file://(Path):1:1:1:1 | (Path) | signature | file://Argument[0]:1:1:1:1 | Argument[0] | input | file://:1:1:1:1 | | output | file://false:1:1:1:1 | false | isVarargsArray | file://sinkModel:1:1:1:1 | sinkModel | extensibleType |
| Test.java:30:4:30:9 | source | path-injection\nrelated locations: $@, $@, $@.\nmetadata: $@, $@, $@, $@, $@, $@, $@, $@, $@. | Test.java:29:3:33:3 | copy(...) | CallContext | Test.java:30:4:30:9 | source | MethodDoc | Test.java:30:4:30:9 | source | ClassDoc | file://java.nio.file:1:1:1:1 | java.nio.file | package | file://Files:1:1:1:1 | Files | type | file://false:1:1:1:1 | false | subtypes | file://copy:1:1:1:1 | copy | name | file://(Path,Path,CopyOption[]):1:1:1:1 | (Path,Path,CopyOption[]) | signature | file://Argument[0]:1:1:1:1 | Argument[0] | input | file://:1:1:1:1 | | output | file://false:1:1:1:1 | false | isVarargsArray | file://sinkModel:1:1:1:1 | sinkModel | extensibleType |
| Test.java:31:4:31:9 | target | path-injection\nrelated locations: $@, $@, $@.\nmetadata: $@, $@, $@, $@, $@, $@, $@, $@, $@. | Test.java:29:3:33:3 | copy(...) | CallContext | Test.java:31:4:31:9 | target | MethodDoc | Test.java:31:4:31:9 | target | ClassDoc | file://java.nio.file:1:1:1:1 | java.nio.file | package | file://Files:1:1:1:1 | Files | type | file://false:1:1:1:1 | false | subtypes | file://copy:1:1:1:1 | copy | name | file://(Path,Path,CopyOption[]):1:1:1:1 | (Path,Path,CopyOption[]) | signature | file://Argument[1]:1:1:1:1 | Argument[1] | input | file://:1:1:1:1 | | output | file://false:1:1:1:1 | false | isVarargsArray | file://sinkModel:1:1:1:1 | sinkModel | extensibleType |
| Test.java:38:4:38:11 | openPath | path-injection\nrelated locations: $@, $@, $@.\nmetadata: $@, $@, $@, $@, $@, $@, $@, $@, $@. | Test.java:37:10:39:3 | newInputStream(...) | CallContext | Test.java:38:4:38:11 | openPath | MethodDoc | Test.java:38:4:38:11 | openPath | ClassDoc | file://java.nio.file:1:1:1:1 | java.nio.file | package | file://Files:1:1:1:1 | Files | type | file://false:1:1:1:1 | false | subtypes | file://newInputStream:1:1:1:1 | newInputStream | name | file://(Path,OpenOption[]):1:1:1:1 | (Path,OpenOption[]) | signature | file://Argument[0]:1:1:1:1 | Argument[0] | input | file://:1:1:1:1 | | output | file://false:1:1:1:1 | false | isVarargsArray | file://sinkModel:1:1:1:1 | sinkModel | extensibleType |
| Test.java:66:3:66:20 | getInputStream(...) | remote\nrelated locations: $@, $@, $@.\nmetadata: $@, $@, $@, $@, $@, $@, $@, $@, $@. | Test.java:66:3:66:20 | getInputStream(...) | CallContext | Test.java:66:3:66:20 | getInputStream(...) | MethodDoc | Test.java:66:3:66:20 | getInputStream(...) | ClassDoc | file://java.net:1:1:1:1 | java.net | package | file://URLConnection:1:1:1:1 | URLConnection | type | file://true:1:1:1:1 | true | subtypes | file://getInputStream:1:1:1:1 | getInputStream | name | file://():1:1:1:1 | () | signature | file://:1:1:1:1 | | input | file://ReturnValue:1:1:1:1 | ReturnValue | output | file://false:1:1:1:1 | false | isVarargsArray | file://sourceModel:1:1:1:1 | sourceModel | extensibleType |
| Test.java:90:28:90:28 | p | path-injection\nrelated locations: $@, $@, $@.\nmetadata: $@, $@, $@, $@, $@, $@, $@, $@, $@. | Test.java:90:4:90:29 | createDirectories(...) | CallContext | Test.java:90:28:90:28 | p | MethodDoc | Test.java:90:28:90:28 | p | ClassDoc | file://java.nio.file:1:1:1:1 | java.nio.file | package | file://Files:1:1:1:1 | Files | type | file://false:1:1:1:1 | false | subtypes | file://createDirectories:1:1:1:1 | createDirectories | name | file://(Path,FileAttribute[]):1:1:1:1 | (Path,FileAttribute[]) | signature | file://Argument[0]:1:1:1:1 | Argument[0] | input | file://:1:1:1:1 | | output | file://false:1:1:1:1 | false | isVarargsArray | file://sinkModel:1:1:1:1 | sinkModel | extensibleType |
| Test.java:94:4:94:4 | p | path-injection\nrelated locations: $@, $@, $@.\nmetadata: $@, $@, $@, $@, $@, $@, $@, $@, $@. | Test.java:93:3:95:3 | delete(...) | CallContext | Test.java:94:4:94:4 | p | MethodDoc | Test.java:94:4:94:4 | p | ClassDoc | file://java.nio.file:1:1:1:1 | java.nio.file | package | file://Files:1:1:1:1 | Files | type | file://false:1:1:1:1 | false | subtypes | file://delete:1:1:1:1 | delete | name | file://(Path):1:1:1:1 | (Path) | signature | file://Argument[0]:1:1:1:1 | Argument[0] | input | file://:1:1:1:1 | | output | file://false:1:1:1:1 | false | isVarargsArray | file://sinkModel:1:1:1:1 | sinkModel | extensibleType |
| Test.java:98:4:98:4 | p | path-injection\nrelated locations: $@, $@, $@.\nmetadata: $@, $@, $@, $@, $@, $@, $@, $@, $@. | Test.java:97:3:99:3 | deleteIfExists(...) | CallContext | Test.java:98:4:98:4 | p | MethodDoc | Test.java:98:4:98:4 | p | ClassDoc | file://java.nio.file:1:1:1:1 | java.nio.file | package | file://Files:1:1:1:1 | Files | type | file://false:1:1:1:1 | false | subtypes | file://deleteIfExists:1:1:1:1 | deleteIfExists | name | file://(Path):1:1:1:1 | (Path) | signature | file://Argument[0]:1:1:1:1 | Argument[0] | input | file://:1:1:1:1 | | output | file://false:1:1:1:1 | false | isVarargsArray | file://sinkModel:1:1:1:1 | sinkModel | extensibleType |

View File

@@ -17,8 +17,8 @@ class Test {
public static void main(String[] args) throws Exception {
AtomicReference<String> reference = new AtomicReference<>(); // uninteresting (parameterless constructor)
reference.set( // $ sinkModel=set(Object):Argument[this]
args[0] // not a sink candidate (modeled as a flow step)
); // not a source candidate (return type is void)
args[0] // $ negativeExample=set(Object):Argument[0] // modeled as a flow step
); // $ negativeExample=set(Object):ReturnValue // return type is void
}
public static void callSupplier(Supplier<String> supplier) {
@@ -42,20 +42,20 @@ class Test {
public static InputStream getInputStream(String openPath) throws Exception {
return Test.getInputStream( // the call is not a source candidate (argument to local call)
Paths.get(
openPath // not a sink candidate (argument to local call)
openPath // $ negativeExample=get(String,String[]):Argument[0] // modeled as a flow step
) // $ sourceModel=get(String,String[]):ReturnValue
);
}
public static int compareFiles(File f1, File f2) {
return f1.compareTo(
f2 // negative sink example (modeled as not a sink)
); // the call is a negative source candidate (sanitizer)
return f1.compareTo( // $ negativeExample=compareTo(File):Argument[this]
f2 // $ negativeExample=compareTo(File):Argument[0] // modeled as not a sink
); // $ negativeExample=compareTo(File):ReturnValue // return type is int
}
public static void FilesWalkExample(Path p, FileVisitOption o) throws Exception {
Files.walk(
p, // negative sink example (modeled as a taint step)
p, // $ negativeExample=walk(Path,FileVisitOption[]):Argument[0] // modeled as a flow step
o, // the implicit varargs array is a candidate, annotated on the last line of the call
o // not a candidate (only the first arg corresponding to a varargs array
// is extracted)
@@ -87,15 +87,15 @@ class TaskUtils {
class MoreTests {
public static void FilesListExample(Path p) throws Exception {
Files.list(
Files.createDirectories(p) // $ sourceModel=createDirectories(Path,FileAttribute[]):ReturnValue // not a sink candidate (modeled as a taint step)
Files.createDirectories(p) // $ sourceModel=createDirectories(Path,FileAttribute[]):ReturnValue negativeExample=list(Path):Argument[0] // modeled as a flow step
); // $ sourceModel=list(Path):ReturnValue
Files.delete(
p // $ sinkModel=delete(Path):Argument[0]
); // not a source candidate (return type is void)
); // $ negativeExample=delete(Path):ReturnValue // return type is void
Files.deleteIfExists(
p // $ sinkModel=deleteIfExists(Path):Argument[0]
); // not a source candidate (return type is boolean)
); // $ negativeExample=deleteIfExists(Path):ReturnValue // return type is boolean
}
}