hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2026-04-30 19:26:02 +02:00
Code Issues Packages Projects Releases Wiki Activity

Python: Rewrite call-graph tests to be inline expectation (2/2)

Browse Source 
I ported the predicates showing difference between points-to and
type-tracking, since it's helpful to see the list of differences,
instead of having to parse expectations!
This commit is contained in:
Rasmus Wriedt Larsen
2022-06-24 16:13:32 +02:00
parent 4caaa3a396
commit 6d9745e5c3
13 changed files with 48 additions and 292 deletions
Download Patch File Download Diff File Expand all files Collapse all files

147
python/ql/test/experimental/library-tests/CallGraph/CallGraphTest.qll
View File

@@ -1,147 +0,0 @@
import python
/** Gets the comment on the line above `ast` */
Comment commentFor(AstNode ast) {
exists(int line | line = ast.getLocation().getStartLine() - 1 |
result
.getLocation()
.hasLocationInfo(ast.getLocation().getFile().getAbsolutePath(), line, _, line, _)
)
}
/** Gets the value from `tag:value` in the comment for `ast` */
string getAnnotation(AstNode ast, string tag) {
exists(Comment comment, string match, string theRegex |
theRegex = "([\\w]+):([\\w.]+)" and
comment = commentFor(ast) and
match = comment.getText().regexpFind(theRegex, _, _) and
tag = match.regexpCapture(theRegex, 1) and
result = match.regexpCapture(theRegex, 2)
)
}
/** Gets a callable annotated with `name:name` */
Function annotatedCallable(string name) { name = getAnnotation(result, "name") }
/** Gets a call annotated with `calls:name` */
Call annotatedCall(string name) { name = getAnnotation(result, "calls") }
predicate missingAnnotationForCallable(string name, Call call) {
call = annotatedCall(name) and
not exists(annotatedCallable(name))
}
predicate nonUniqueAnnotationForCallable(string name, Function callable) {
strictcount(annotatedCallable(name)) > 1 and
callable = annotatedCallable(name)
}
predicate missingAnnotationForCall(string name, Function callable) {
not exists(annotatedCall(name)) and
callable = annotatedCallable(name)
}
/** There is an obvious problem with the annotation `name` */
predicate nameInErrorState(string name) {
missingAnnotationForCallable(name, _)
or
nonUniqueAnnotationForCallable(name, _)
or
missingAnnotationForCall(name, _)
}
/** Source code has annotation with `name` showing that `call` will call `callable` */
predicate annotatedCallEdge(string name, Call call, Function callable) {
not nameInErrorState(name) and
call = annotatedCall(name) and
callable = annotatedCallable(name)
}
// ------------------------- Annotation debug query predicates -------------------------
query predicate debug_missingAnnotationForCallable(Call call, string message) {
exists(string name |
message =
"This call is annotated with '" + name +
"', but no callable with that annotation was extracted. Please fix." and
missingAnnotationForCallable(name, call)
)
}
query predicate debug_nonUniqueAnnotationForCallable(Function callable, string message) {
exists(string name |
message = "Multiple callables are annotated with '" + name + "'. Please fix." and
nonUniqueAnnotationForCallable(name, callable)
)
}
query predicate debug_missingAnnotationForCall(Function callable, string message) {
exists(string name |
message =
"This callable is annotated with '" + name +
"', but no call with that annotation was extracted. Please fix." and
missingAnnotationForCall(name, callable)
)
}
// ------------------------- Call Graph resolution -------------------------
private newtype TCallGraphResolver =
TPointsToResolver() or
TTypeTrackerResolver()
/** A method of call graph resolution */
abstract class CallGraphResolver extends TCallGraphResolver {
abstract predicate callEdge(Call call, Function callable);
/**
* Holds if annotations show that `call` will call `callable`,
* but our call graph resolver was not able to figure that out
*/
predicate expectedCallEdgeNotFound(Call call, Function callable) {
annotatedCallEdge(_, call, callable) and
not this.callEdge(call, callable)
}
/**
* Holds if there are no annotations that show that `call` will call `callable` (where at least one of these are annotated),
* but the call graph resolver claims that `call` will call `callable`
*/
predicate unexpectedCallEdgeFound(Call call, Function callable, string message) {
this.callEdge(call, callable) and
not annotatedCallEdge(_, call, callable) and
(
exists(string name |
message = "Call resolved to the callable named '" + name + "' but was not annotated as such" and
callable = annotatedCallable(name) and
not nameInErrorState(name)
)
or
exists(string name |
message = "Annotated call resolved to unannotated callable" and
call = annotatedCall(name) and
not nameInErrorState(name) and
not exists( | callable = annotatedCallable(_))
)
)
}
string toString() { result = "CallGraphResolver" }
}
/** A call graph resolver based on the existing points-to analysis */
class PointsToResolver extends CallGraphResolver, TPointsToResolver {
override predicate callEdge(Call call, Function callable) {
exists(PythonFunctionValue funcValue |
funcValue.getScope() = callable and
call = funcValue.getACall().getNode()
)
}
override string toString() { result = "PointsToResolver" }
}
/** A call graph resolved based on Type Trackers */
class TypeTrackerResolver extends CallGraphResolver, TTypeTrackerResolver {
override predicate callEdge(Call call, Function callable) { none() }
override string toString() { result = "TypeTrackerResolver" }
}

18
python/ql/test/experimental/library-tests/CallGraph/InlineCallGraphTest.expected
View File

@@ -2,3 +2,21 @@ failures
debug_callableNotUnique
| code/class_advanced.py:18:5:18:18 | Function arg | Qualified function name 'B.arg' is not unique. Please fix. |
| code/class_advanced.py:23:5:23:25 | Function arg | Qualified function name 'B.arg' is not unique. Please fix. |
pointsTo_found_typeTracker_notFound
| code/class_simple.py:28:1:28:15 | ControlFlowNode for Attribute() | A.some_method |
| code/class_simple.py:30:1:30:21 | ControlFlowNode for Attribute() | A.some_staticmethod |
| code/class_simple.py:32:1:32:20 | ControlFlowNode for Attribute() | A.some_classmethod |
| code/class_simple.py:35:1:35:21 | ControlFlowNode for Attribute() | A.some_staticmethod |
| code/class_simple.py:37:1:37:20 | ControlFlowNode for Attribute() | A.some_classmethod |
| code/runtime_decision.py:18:1:18:6 | ControlFlowNode for func() | rd_bar |
| code/runtime_decision.py:18:1:18:6 | ControlFlowNode for func() | rd_foo |
| code/runtime_decision.py:26:1:26:7 | ControlFlowNode for func2() | rd_bar |
| code/runtime_decision.py:26:1:26:7 | ControlFlowNode for func2() | rd_foo |
| code/simple.py:15:1:15:5 | ControlFlowNode for foo() | foo |
| code/simple.py:16:1:16:14 | ControlFlowNode for indirect_foo() | foo |
| code/simple.py:17:1:17:5 | ControlFlowNode for bar() | bar |
| code/simple.py:18:1:18:5 | ControlFlowNode for lam() | lambda[simple.py:12:7] |
| code/underscore_prefix_func_name.py:18:5:18:19 | ControlFlowNode for some_function() | some_function |
| code/underscore_prefix_func_name.py:21:5:21:19 | ControlFlowNode for some_function() | some_function |
| code/underscore_prefix_func_name.py:24:1:24:21 | ControlFlowNode for _works_since_called() | _works_since_called |
typeTracker_found_pointsTo_notFound

39
python/ql/test/experimental/library-tests/CallGraph/InlineCallGraphTest.ql
View File

@@ -28,22 +28,41 @@ class CallGraphTest extends InlineExpectationsTest {
|
location = call.getLocation() and
element = call.toString() and
(
// note: `target.getQualifiedName` for Lambdas is just "lambda", so is not very useful :|
not target.isLambda() and
value = target.getQualifiedName()
or
target.isLambda() and
value =
"lambda[" + target.getLocation().getFile().getShortName() + ":" +
target.getLocation().getStartLine() + ":" + target.getLocation().getStartColumn() + "]"
)
value = betterQualName(target)
)
}
}
bindingset[func]
string betterQualName(Function func) {
// note: `target.getQualifiedName` for Lambdas is just "lambda", so is not very useful :|
not func.isLambda() and
result = func.getQualifiedName()
or
func.isLambda() and
result =
"lambda[" + func.getLocation().getFile().getShortName() + ":" +
func.getLocation().getStartLine() + ":" + func.getLocation().getStartColumn() + "]"
}
query predicate debug_callableNotUnique(Function callable, string message) {
exists(Function f | f != callable and f.getQualifiedName() = callable.getQualifiedName()) and
message =
"Qualified function name '" + callable.getQualifiedName() + "' is not unique. Please fix."
}
query predicate pointsTo_found_typeTracker_notFound(CallNode call, string qualname) {
exists(Function target |
pointsToCallEdge(call, target) and
not typeTrackerCallEdge(call, target) and
qualname = betterQualName(target)
)
}
query predicate typeTracker_found_pointsTo_notFound(CallNode call, string qualname) {
exists(Function target |
not pointsToCallEdge(call, target) and
typeTrackerCallEdge(call, target) and
qualname = betterQualName(target)
)
}

6
python/ql/test/experimental/library-tests/CallGraph/PointsTo.expected
View File

@@ -1,6 +0,0 @@
debug_missingAnnotationForCallable
debug_nonUniqueAnnotationForCallable
debug_missingAnnotationForCall
expectedCallEdgeNotFound
| code/underscore_prefix_func_name.py:16:5:16:19 | some_function() | code/underscore_prefix_func_name.py:10:1:10:20 | Function some_function |
unexpectedCallEdgeFound

10
python/ql/test/experimental/library-tests/CallGraph/PointsTo.ql
View File

@@ -1,10 +0,0 @@
import python
import CallGraphTest
query predicate expectedCallEdgeNotFound(Call call, Function callable) {
any(PointsToResolver r).expectedCallEdgeNotFound(call, callable)
}
query predicate unexpectedCallEdgeFound(Call call, Function callable, string message) {
any(PointsToResolver r).unexpectedCallEdgeFound(call, callable, message)
}

38
python/ql/test/experimental/library-tests/CallGraph/README.md
View File

@@ -1,38 +0,0 @@
# Call Graph Tests
A small testing framework for our call graph resolution. It relies on manual annotation of calls and callables, **and will only include output if something is wrong**. For example, if we are not able to resolve that the `foo()` call will call the `foo` function, that should give an alert.
```py
# name:foo
def foo():
pass
# calls:foo
foo()
```
This is greatly inspired by [`CallGraphs/AnnotatedTest`](https://github.com/github/codeql/blob/696d19cb1440b6f6a75c6a2c1319e18860ceb436/javascript/ql/test/library-tests/CallGraphs/AnnotatedTest/Test.ql) from JavaScript.
IMPORTANT: Names used in annotations are not scoped, so must be unique globally. (this is a bit annoying, but makes things simple). If multiple identical annotations are used, an error message will be output.
Important files:
- `CallGraphTest.qll`: main code to find annotated calls/callables and setting everything up.
- `PointsTo.ql`: results when using points-to for call graph resolution.
- `TypeTracker.ql`: results when using TypeTracking for call graph resolution.
- `Relative.ql`: differences between using points-to and TypeTracking.
- `code/` contains the actual Python code we test against (included by `test.py`).
All queries will also execute some `debug_*` predicates. These highlight any obvious problems with the annotation setup, and so there should never be any results committed. To show that this works as expected, see the [CallGraph-xfail](../CallGraph-xfail/) which uses symlinked versions of the files in this directory (can't include as subdir, so has to be a sibling).
## `options` file
If the value for `--max-import-depth` is set so that `import random` will extract `random.py` from the standard library, BUT NO transitive imports are extracted, then points-to analysis will fail to handle the following snippet.
```py
import random
if random.random() < 0.5:
func = foo
else:
func = bar
func()
```

20
python/ql/test/experimental/library-tests/CallGraph/Relative.expected
View File

@@ -1,20 +0,0 @@
debug_missingAnnotationForCallable
debug_nonUniqueAnnotationForCallable
debug_missingAnnotationForCall
pointsTo_found_typeTracker_notFound
| code/class_simple.py:28:1:28:15 | Attribute() | code/class_simple.py:8:5:8:26 | Function some_method |
| code/class_simple.py:30:1:30:21 | Attribute() | code/class_simple.py:13:5:13:28 | Function some_staticmethod |
| code/class_simple.py:32:1:32:20 | Attribute() | code/class_simple.py:18:5:18:30 | Function some_classmethod |
| code/class_simple.py:35:1:35:21 | Attribute() | code/class_simple.py:13:5:13:28 | Function some_staticmethod |
| code/class_simple.py:37:1:37:20 | Attribute() | code/class_simple.py:18:5:18:30 | Function some_classmethod |
| code/runtime_decision.py:21:1:21:6 | func() | code/runtime_decision.py:8:1:8:13 | Function rd_foo |
| code/runtime_decision.py:21:1:21:6 | func() | code/runtime_decision.py:12:1:12:13 | Function rd_bar |
| code/runtime_decision.py:30:1:30:7 | func2() | code/runtime_decision.py:8:1:8:13 | Function rd_foo |
| code/runtime_decision.py:30:1:30:7 | func2() | code/runtime_decision.py:12:1:12:13 | Function rd_bar |
| code/simple.py:19:1:19:5 | foo() | code/simple.py:2:1:2:10 | Function foo |
| code/simple.py:21:1:21:14 | indirect_foo() | code/simple.py:2:1:2:10 | Function foo |
| code/simple.py:23:1:23:5 | bar() | code/simple.py:10:1:10:10 | Function bar |
| code/simple.py:25:1:25:5 | lam() | code/simple.py:15:7:15:36 | Function lambda |
| code/underscore_prefix_func_name.py:21:5:21:19 | some_function() | code/underscore_prefix_func_name.py:10:1:10:20 | Function some_function |
| code/underscore_prefix_func_name.py:25:5:25:19 | some_function() | code/underscore_prefix_func_name.py:10:1:10:20 | Function some_function |
pointsTo_notFound_typeTracker_found

14
python/ql/test/experimental/library-tests/CallGraph/Relative.ql
View File

@@ -1,14 +0,0 @@
import python
import CallGraphTest
query predicate pointsTo_found_typeTracker_notFound(Call call, Function callable) {
annotatedCallEdge(_, call, callable) and
any(PointsToResolver r).callEdge(call, callable) and
not any(TypeTrackerResolver r).callEdge(call, callable)
}
query predicate pointsTo_notFound_typeTracker_found(Call call, Function callable) {
annotatedCallEdge(_, call, callable) and
not any(PointsToResolver r).callEdge(call, callable) and
any(TypeTrackerResolver r).callEdge(call, callable)
}

21
python/ql/test/experimental/library-tests/CallGraph/TypeTracker.expected
View File

@@ -1,21 +0,0 @@
debug_missingAnnotationForCallable
debug_nonUniqueAnnotationForCallable
debug_missingAnnotationForCall
expectedCallEdgeNotFound
| code/class_simple.py:28:1:28:15 | Attribute() | code/class_simple.py:8:5:8:26 | Function some_method |
| code/class_simple.py:30:1:30:21 | Attribute() | code/class_simple.py:13:5:13:28 | Function some_staticmethod |
| code/class_simple.py:32:1:32:20 | Attribute() | code/class_simple.py:18:5:18:30 | Function some_classmethod |
| code/class_simple.py:35:1:35:21 | Attribute() | code/class_simple.py:13:5:13:28 | Function some_staticmethod |
| code/class_simple.py:37:1:37:20 | Attribute() | code/class_simple.py:18:5:18:30 | Function some_classmethod |
| code/runtime_decision.py:21:1:21:6 | func() | code/runtime_decision.py:8:1:8:13 | Function rd_foo |
| code/runtime_decision.py:21:1:21:6 | func() | code/runtime_decision.py:12:1:12:13 | Function rd_bar |
| code/runtime_decision.py:30:1:30:7 | func2() | code/runtime_decision.py:8:1:8:13 | Function rd_foo |
| code/runtime_decision.py:30:1:30:7 | func2() | code/runtime_decision.py:12:1:12:13 | Function rd_bar |
| code/simple.py:19:1:19:5 | foo() | code/simple.py:2:1:2:10 | Function foo |
| code/simple.py:21:1:21:14 | indirect_foo() | code/simple.py:2:1:2:10 | Function foo |
| code/simple.py:23:1:23:5 | bar() | code/simple.py:10:1:10:10 | Function bar |
| code/simple.py:25:1:25:5 | lam() | code/simple.py:15:7:15:36 | Function lambda |
| code/underscore_prefix_func_name.py:16:5:16:19 | some_function() | code/underscore_prefix_func_name.py:10:1:10:20 | Function some_function |
| code/underscore_prefix_func_name.py:21:5:21:19 | some_function() | code/underscore_prefix_func_name.py:10:1:10:20 | Function some_function |
| code/underscore_prefix_func_name.py:25:5:25:19 | some_function() | code/underscore_prefix_func_name.py:10:1:10:20 | Function some_function |
unexpectedCallEdgeFound

10
python/ql/test/experimental/library-tests/CallGraph/TypeTracker.ql
View File

@@ -1,10 +0,0 @@
import python
import CallGraphTest
query predicate expectedCallEdgeNotFound(Call call, Function callable) {
any(TypeTrackerResolver r).expectedCallEdgeNotFound(call, callable)
}
query predicate unexpectedCallEdgeFound(Call call, Function callable, string message) {
any(TypeTrackerResolver r).unexpectedCallEdgeFound(call, callable, message)
}

4
python/ql/test/experimental/library-tests/CallGraph/code/runtime_decision.py
View File

@@ -4,11 +4,9 @@ import random
# hmm, annoying that you have to keep names unique across files :|
# since I like to use foo and bar ALL the time :D
# name:rd_foo
def rd_foo():
print('rd_foo')
# name:rd_bar
def rd_bar():
print('rd_bar')
@@ -17,7 +15,6 @@ if len(sys.argv) >= 2 and not sys.argv[1] in ['0', 'False', 'false']:
else:
func = rd_bar
# calls:rd_foo calls:rd_bar
func() # $ pt=rd_foo pt=rd_bar
# Random doesn't work with points-to :O
@@ -26,5 +23,4 @@ if random.random() < 0.5:
else:
func2 = rd_bar
# calls:rd_foo calls:rd_bar
func2() # $ pt=rd_foo pt=rd_bar

9
python/ql/test/experimental/library-tests/CallGraph/code/simple.py
View File

@@ -1,4 +1,3 @@
# name:foo
def foo():
print("foo called")
@@ -6,22 +5,16 @@ def foo():
indirect_foo = foo
# name:bar
def bar():
print("bar called")
# name:lam
lam = lambda: print("lambda called")
# calls:foo
foo() # $ pt=foo
# calls:foo
indirect_foo() # $ pt=foo
# calls:bar
bar() # $ pt=bar
# calls:lam
lam() # $ pt=lambda[simple.py:15:7]
lam() # $ pt=lambda[simple.py:12:7]
# python -m trace --trackcalls simple.py

4
python/ql/test/experimental/library-tests/CallGraph/code/underscore_prefix_func_name.py
View File

@@ -6,22 +6,18 @@
# points-to information about the `open` call in
# https://google-gruyere.appspot.com/code/gruyere.py on line 227
# name:some_function
def some_function():
print('some_function')
def _ignored():
print('_ignored')
# calls:some_function
some_function()
def _works_since_called():
print('_works_since_called')
# calls:some_function
some_function() # $ pt=some_function
def works_even_though_not_called():
# calls:some_function
some_function() # $ pt=some_function
globals()['_ignored']()