hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2025-12-24 12:46:34 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #14848 from hvitved/python/shared-type-tracking

Browse Source 
Python: Adopt shared type tracking library
This commit is contained in:
yoff
2023-12-18 21:14:42 +01:00
committed by GitHub
parent 41c49ae05b a776132a10
commit e0c027f13c
24 changed files with 503 additions and 731 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
python/ql/lib/change-notes/2023-11-21-new-type-tracking-lib.md Normal file
View File

@@ -0,0 +1,4 @@
---
category: minorAnalysis
---
* Python now makes use of the shared type tracking library, exposed as `semmle.python.dataflow.new.TypeTracking`. The existing type tracking library, `semmle.python.dataflow.new.TypeTracker`, has consequently been deprecated.

4
python/ql/lib/experimental/cryptography/utils/CallCfgNodeWithTarget.qll
View File

@@ -4,9 +4,9 @@
*/
import python
private import semmle.python.dataflow.new.internal.TypeTrackerSpecific
private import semmle.python.dataflow.new.internal.TypeTrackingImpl
private import semmle.python.ApiGraphs
class CallCfgNodeWithTarget extends DataFlow::Node instanceof DataFlow::CallCfgNode {
DataFlow::Node getTarget() { returnStep(result, this) }
DataFlow::Node getTarget() { TypeTrackingInput::returnStep(result, this) }
}

20
python/ql/lib/semmle/python/dataflow/new/TypeTracker.qll
View File

@@ -1,4 +1,6 @@
/**
* DEPRECATED: Use `semmle.python.dataflow.new.TypeTracking` instead.
*
* This file acts as a wrapper for `internal.TypeTracker`, exposing some of the functionality with
* names that are more appropriate for Python.
*/
@@ -8,12 +10,14 @@ private import internal.TypeTracker as Internal
private import internal.TypeTrackerSpecific as InternalSpecific
/** A string that may appear as the name of an attribute or access path. */
class AttributeName = InternalSpecific::TypeTrackerContent;
deprecated class AttributeName = InternalSpecific::TypeTrackerContent;
/** An attribute name, or the empty string (representing no attribute). */
class OptionalAttributeName = InternalSpecific::OptionalTypeTrackerContent;
deprecated class OptionalAttributeName = InternalSpecific::OptionalTypeTrackerContent;
/**
* DEPRECATED: Use `semmle.python.dataflow.new.TypeTracking` instead.
*
* The summary of the steps needed to track a value to a given dataflow node.
*
* This can be used to track objects that implement a certain API in order to
@@ -40,7 +44,7 @@ class OptionalAttributeName = InternalSpecific::OptionalTypeTrackerContent;
* `t = t2.step(myType(t2), result)`. If you additionally want to track individual
* intra-procedural steps, use `t = t2.smallstep(myCallback(t2), result)`.
*/
class TypeTracker extends Internal::TypeTracker {
deprecated class TypeTracker extends Internal::TypeTracker {
/**
* Holds if this is the starting point of type tracking, and the value starts in the attribute named `attrName`.
* The type tracking only ends after the attribute has been loaded.
@@ -55,12 +59,12 @@ class TypeTracker extends Internal::TypeTracker {
string getAttr() { result = this.getContent() }
}
module TypeTracker = Internal::TypeTracker;
deprecated module TypeTracker = Internal::TypeTracker;
class StepSummary = Internal::StepSummary;
deprecated class StepSummary = Internal::StepSummary;
module StepSummary = Internal::StepSummary;
deprecated module StepSummary = Internal::StepSummary;
class TypeBackTracker = Internal::TypeBackTracker;
deprecated class TypeBackTracker = Internal::TypeBackTracker;
module TypeBackTracker = Internal::TypeBackTracker;
deprecated module TypeBackTracker = Internal::TypeBackTracker;

56
python/ql/lib/semmle/python/dataflow/new/TypeTracking.qll Normal file
View File

@@ -0,0 +1,56 @@
/**
* Provides classes and predicates for simple data-flow reachability suitable
* for tracking types.
*/
private import internal.TypeTrackingImpl as Impl
import Impl::Shared::TypeTracking<Impl::TypeTrackingInput>
/** A string that may appear as the name of an attribute or access path. */
class AttributeName = Impl::TypeTrackingInput::Content;
/**
* A summary of the steps needed to track a value to a given dataflow node.
*
* This can be used to track objects that implement a certain API in order to
* recognize calls to that API. Note that type-tracking does not by itself provide a
* source/sink relation, that is, it may determine that a node has a given type,
* but it won't determine where that type came from.
*
* It is recommended that all uses of this type are written in the following form,
* for tracking some type `myType`:
* ```ql
* Node myType(TypeTracker tt) {
* tt.start() and
* result = < source of myType >
* or
* exists(TypeTracker tt2 |
* tt = tt2.step(myType(tt2), result)
* )
* }
*
* Node myType() { myType(TypeTracker::end()).flowsTo(result) }
* ```
*
* If you want to track individual intra-procedural steps, use `tt2.smallstep`
* instead of `tt2.step`.
*/
class TypeTracker extends Impl::TypeTracker {
/**
* Holds if this is the starting point of type tracking, and the value starts in the attribute named `attrName`.
* The type tracking only ends after the attribute has been loaded.
*/
predicate startInAttr(string attrName) { this.startInContent(attrName) }
/**
* INTERNAL. DO NOT USE.
*
* Gets the attribute associated with this type tracker.
*/
string getAttr() {
result = this.getContent().asSome()
or
this.getContent().isNone() and
result = ""
}
}

2
python/ql/lib/semmle/python/dataflow/new/internal/DataFlowDispatch.qll
View File

@@ -37,7 +37,7 @@ private import DataFlowPublic
private import DataFlowPrivate
private import FlowSummaryImpl as FlowSummaryImpl
private import semmle.python.internal.CachedStages
private import semmle.python.dataflow.new.internal.TypeTracker::CallGraphConstruction as CallGraphConstruction
private import semmle.python.dataflow.new.internal.TypeTrackingImpl::CallGraphConstruction as CallGraphConstruction
newtype TParameterPosition =
/** Used for `self` in methods, and `cls` in classmethods. */

2
python/ql/lib/semmle/python/dataflow/new/internal/DataFlowPublic.qll
View File

@@ -4,7 +4,7 @@
private import python
private import DataFlowPrivate
import semmle.python.dataflow.new.TypeTracker
import semmle.python.dataflow.new.TypeTracking
import Attributes
import LocalSources
private import semmle.python.essa.SsaCompute

2
python/ql/lib/semmle/python/dataflow/new/internal/ImportResolution.qll
View File

@@ -7,7 +7,7 @@
private import python
private import semmle.python.dataflow.new.DataFlow
private import semmle.python.dataflow.new.internal.ImportStar
private import semmle.python.dataflow.new.TypeTracker
private import semmle.python.dataflow.new.TypeTracking
private import semmle.python.dataflow.new.internal.DataFlowPrivate
/**

6
python/ql/lib/semmle/python/dataflow/new/internal/LocalSources.qll
View File

@@ -72,6 +72,8 @@ class LocalSourceNode extends Node {
// We include all scope entry definitions, as these act as the local source within the scope they
// enter.
this.asCfgNode() = any(ScopeEntryDefinition def).getDefiningNode()
or
this instanceof ParameterNode
}
/** Holds if this `LocalSourceNode` can flow to `nodeTo` in one or more local flow steps. */
@@ -151,7 +153,7 @@ class LocalSourceNode extends Node {
* See `TypeBackTracker` for more details about how to use this.
*/
pragma[inline]
LocalSourceNode backtrack(TypeBackTracker t2, TypeBackTracker t) { t2 = t.step(result, this) }
LocalSourceNode backtrack(TypeBackTracker t2, TypeBackTracker t) { t = t2.step(result, this) }
}
/**
@@ -238,7 +240,7 @@ private module Cached {
* Helper predicate for `hasLocalSource`. Removes any steps go to module variable reads, as these
* are already local source nodes in their own right.
*/
cached
pragma[nomagic]
private predicate localSourceFlowStep(Node nodeFrom, Node nodeTo) {
simpleLocalFlowStep(nodeFrom, nodeTo) and
not nodeTo = any(ModuleVariableNode v).getARead()

412
python/ql/lib/semmle/python/dataflow/new/internal/SummaryTypeTracker.qll
View File

@@ -1,412 +0,0 @@
/**
* Provides the implementation of type tracking steps through flow summaries.
* To use this, you must implement the `Input` signature. You can then use the predicates in the `Output`
* signature to implement the predicates of the same names inside `TypeTrackerSpecific.qll`.
*/
/** The classes and predicates needed to generate type-tracking steps from summaries. */
signature module Input {
// Dataflow nodes
class Node;
// Content
class TypeTrackerContent;
class TypeTrackerContentFilter;
// Relating content and filters
/**
* Gets a content filter to use for a `WithoutContent[content]` step, (data is not allowed to be stored in `content`)
* or has no result if
* the step should be treated as ordinary flow.
*
* `WithoutContent` is often used to perform strong updates on individual collection elements, but for
* type-tracking this is rarely beneficial and quite expensive. However, `WithoutContent` can be quite useful
* for restricting the type of an object, and in these cases we translate it to a filter.
*/
TypeTrackerContentFilter getFilterFromWithoutContentStep(TypeTrackerContent content);
/**
* Gets a content filter to use for a `WithContent[content]` step, (data must be stored in `content`)
* or has no result if
* the step cannot be handled by type-tracking.
*
* `WithContent` is often used to perform strong updates on individual collection elements (or rather
* to preserve those that didn't get updated). But for type-tracking this is rarely beneficial and quite expensive.
* However, `WithContent` can be quite useful for restricting the type of an object, and in these cases we translate it to a filter.
*/
TypeTrackerContentFilter getFilterFromWithContentStep(TypeTrackerContent content);
// Summaries and their stacks
class SummaryComponent;
class SummaryComponentStack {
SummaryComponent head();
}
/** Gets a singleton stack containing `component`. */
SummaryComponentStack singleton(SummaryComponent component);
/**
* Gets the stack obtained by pushing `head` onto `tail`.
*/
SummaryComponentStack push(SummaryComponent head, SummaryComponentStack tail);
/** Gets a singleton stack representing a return. */
SummaryComponent return();
// Relating content to summaries
/** Gets a summary component for content `c`. */
SummaryComponent content(TypeTrackerContent contents);
/** Gets a summary component where data is not allowed to be stored in `contents`. */
SummaryComponent withoutContent(TypeTrackerContent contents);
/** Gets a summary component where data must be stored in `contents`. */
SummaryComponent withContent(TypeTrackerContent contents);
// Callables
class SummarizedCallable {
predicate propagatesFlow(
SummaryComponentStack input, SummaryComponentStack output, boolean preservesValue
);
}
// Relating nodes to summaries
/**
* Gets a dataflow node respresenting the argument of `call` indicated by `arg`.
*
* Returns the post-update node of the argument when `isPostUpdate` is true.
*/
Node argumentOf(Node call, SummaryComponent arg, boolean isPostUpdate);
/** Gets a dataflow node respresenting the parameter of `callable` indicated by `param`. */
Node parameterOf(Node callable, SummaryComponent param);
/** Gets a dataflow node respresenting the return of `callable` indicated by `return`. */
Node returnOf(Node callable, SummaryComponent return);
// Relating callables to nodes
/** Gets a dataflow node respresenting a call to `callable`. */
Node callTo(SummarizedCallable callable);
}
/**
* The predicates provided by a summary type tracker.
* These are meant to be used in `TypeTrackerSpecific.qll`
* inside the predicates of the same names.
*/
signature module Output<Input I> {
/**
* Holds if there is a level step from `nodeFrom` to `nodeTo`, which does not depend on the call graph.
*/
predicate levelStepNoCall(I::Node nodeFrom, I::Node nodeTo);
/**
* Holds if `nodeTo` is the result of accessing the `content` content of `nodeFrom`.
*/
predicate basicLoadStep(I::Node nodeFrom, I::Node nodeTo, I::TypeTrackerContent content);
/**
* Holds if `nodeFrom` is being written to the `content` content of the object in `nodeTo`.
*/
predicate basicStoreStep(I::Node nodeFrom, I::Node nodeTo, I::TypeTrackerContent content);
/**
* Holds if the `loadContent` of `nodeFrom` is stored in the `storeContent` of `nodeTo`.
*/
predicate basicLoadStoreStep(
I::Node nodeFrom, I::Node nodeTo, I::TypeTrackerContent loadContent,
I::TypeTrackerContent storeContent
);
/**
* Holds if type-tracking should step from `nodeFrom` to `nodeTo` but block flow of contents matched by `filter` through here.
*/
predicate basicWithoutContentStep(
I::Node nodeFrom, I::Node nodeTo, I::TypeTrackerContentFilter filter
);
/**
* Holds if type-tracking should step from `nodeFrom` to `nodeTo` if inside a content matched by `filter`.
*/
predicate basicWithContentStep(
I::Node nodeFrom, I::Node nodeTo, I::TypeTrackerContentFilter filter
);
}
/**
* Implementation of the summary type tracker, that is type tracking through flow summaries.
*/
module SummaryFlow<Input I> implements Output<I> {
pragma[nomagic]
private predicate isNonLocal(I::SummaryComponent component) {
component = I::content(_)
or
component = I::withContent(_)
}
pragma[nomagic]
private predicate hasLoadSummary(
I::SummarizedCallable callable, I::TypeTrackerContent contents, I::SummaryComponentStack input,
I::SummaryComponentStack output
) {
callable.propagatesFlow(I::push(I::content(contents), input), output, true) and
not isNonLocal(input.head()) and
not isNonLocal(output.head())
}
pragma[nomagic]
private predicate hasStoreSummary(
I::SummarizedCallable callable, I::TypeTrackerContent contents, I::SummaryComponentStack input,
I::SummaryComponentStack output
) {
not isNonLocal(input.head()) and
not isNonLocal(output.head()) and
(
callable.propagatesFlow(input, I::push(I::content(contents), output), true)
or
// Allow the input to start with an arbitrary WithoutContent[X].
// Since type-tracking only tracks one content deep, and we're about to store into another content,
// we're already preventing the input from being in a content.
callable
.propagatesFlow(I::push(I::withoutContent(_), input),
I::push(I::content(contents), output), true)
)
}
pragma[nomagic]
private predicate hasLoadStoreSummary(
I::SummarizedCallable callable, I::TypeTrackerContent loadContents,
I::TypeTrackerContent storeContents, I::SummaryComponentStack input,
I::SummaryComponentStack output
) {
callable
.propagatesFlow(I::push(I::content(loadContents), input),
I::push(I::content(storeContents), output), true) and
not isNonLocal(input.head()) and
not isNonLocal(output.head())
}
pragma[nomagic]
private predicate hasWithoutContentSummary(
I::SummarizedCallable callable, I::TypeTrackerContentFilter filter,
I::SummaryComponentStack input, I::SummaryComponentStack output
) {
exists(I::TypeTrackerContent content |
callable.propagatesFlow(I::push(I::withoutContent(content), input), output, true) and
filter = I::getFilterFromWithoutContentStep(content) and
not isNonLocal(input.head()) and
not isNonLocal(output.head()) and
input != output
)
}
pragma[nomagic]
private predicate hasWithContentSummary(
I::SummarizedCallable callable, I::TypeTrackerContentFilter filter,
I::SummaryComponentStack input, I::SummaryComponentStack output
) {
exists(I::TypeTrackerContent content |
callable.propagatesFlow(I::push(I::withContent(content), input), output, true) and
filter = I::getFilterFromWithContentStep(content) and
not isNonLocal(input.head()) and
not isNonLocal(output.head()) and
input != output
)
}
private predicate componentLevelStep(I::SummaryComponent component) {
exists(I::TypeTrackerContent content |
component = I::withoutContent(content) and
not exists(I::getFilterFromWithoutContentStep(content))
)
}
/**
* Gets a data flow `I::Node` corresponding an argument or return value of `call`,
* as specified by `component`. `isOutput` indicates whether the node represents
* an output node or an input node.
*/
bindingset[call, component]
private I::Node evaluateSummaryComponentLocal(
I::Node call, I::SummaryComponent component, boolean isOutput
) {
result = I::argumentOf(call, component, isOutput)
or
component = I::return() and
result = call and
isOutput = true
}
/**
* Holds if `callable` is relevant for type-tracking and we therefore want `stack` to
* be evaluated locally at its call sites.
*/
pragma[nomagic]
private predicate dependsOnSummaryComponentStack(
I::SummarizedCallable callable, I::SummaryComponentStack stack
) {
exists(I::callTo(callable)) and
(
callable.propagatesFlow(stack, _, true)
or
callable.propagatesFlow(_, stack, true)
or
// include store summaries as they may skip an initial step at the input
hasStoreSummary(callable, _, stack, _)
)
or
dependsOnSummaryComponentStackCons(callable, _, stack)
}
pragma[nomagic]
private predicate dependsOnSummaryComponentStackCons(
I::SummarizedCallable callable, I::SummaryComponent head, I::SummaryComponentStack tail
) {
dependsOnSummaryComponentStack(callable, I::push(head, tail))
}
pragma[nomagic]
private predicate dependsOnSummaryComponentStackConsLocal(
I::SummarizedCallable callable, I::SummaryComponent head, I::SummaryComponentStack tail
) {
dependsOnSummaryComponentStackCons(callable, head, tail) and
not isNonLocal(head)
}
pragma[nomagic]
private predicate dependsOnSummaryComponentStackLeaf(
I::SummarizedCallable callable, I::SummaryComponent leaf
) {
dependsOnSummaryComponentStack(callable, I::singleton(leaf))
}
/**
* Gets a data flow I::Node corresponding to the local input or output of `call`
* identified by `stack`, if possible.
*/
pragma[nomagic]
private I::Node evaluateSummaryComponentStackLocal(
I::SummarizedCallable callable, I::Node call, I::SummaryComponentStack stack, boolean isOutput
) {
exists(I::SummaryComponent component |
dependsOnSummaryComponentStackLeaf(callable, component) and
stack = I::singleton(component) and
call = I::callTo(callable) and
result = evaluateSummaryComponentLocal(call, component, isOutput)
)
or
exists(
I::Node prev, I::SummaryComponent head, I::SummaryComponentStack tail, boolean isOutput0
|
prev = evaluateSummaryComponentStackLocal(callable, call, tail, isOutput0) and
dependsOnSummaryComponentStackConsLocal(callable, pragma[only_bind_into](head),
pragma[only_bind_out](tail)) and
stack = I::push(pragma[only_bind_out](head), pragma[only_bind_out](tail))
|
// `Parameter[X]` is only allowed in the output of flow summaries (hence `isOutput = true`),
// however the target of the parameter (e.g. `Argument[Y].Parameter[X]`) should be fetched
// not from a post-update argument node (hence `isOutput0 = false`)
result = I::parameterOf(prev, head) and
isOutput0 = false and
isOutput = true
or
// `ReturnValue` is only allowed in the input of flow summaries (hence `isOutput = false`),
// and the target of the return value (e.g. `Argument[X].ReturnValue`) should be fetched not
// from a post-update argument node (hence `isOutput0 = false`)
result = I::returnOf(prev, head) and
isOutput0 = false and
isOutput = false
or
componentLevelStep(head) and
result = prev and
isOutput = isOutput0
)
}
// Implement Output
predicate levelStepNoCall(I::Node nodeFrom, I::Node nodeTo) {
exists(
I::SummarizedCallable callable, I::Node call, I::SummaryComponentStack input,
I::SummaryComponentStack output
|
callable.propagatesFlow(input, output, true) and
call = I::callTo(callable) and
nodeFrom = evaluateSummaryComponentStackLocal(callable, call, input, false) and
nodeTo = evaluateSummaryComponentStackLocal(callable, call, output, true)
)
}
predicate basicLoadStep(I::Node nodeFrom, I::Node nodeTo, I::TypeTrackerContent content) {
exists(
I::SummarizedCallable callable, I::Node call, I::SummaryComponentStack input,
I::SummaryComponentStack output
|
hasLoadSummary(callable, content, pragma[only_bind_into](input),
pragma[only_bind_into](output)) and
call = I::callTo(callable) and
nodeFrom = evaluateSummaryComponentStackLocal(callable, call, input, false) and
nodeTo = evaluateSummaryComponentStackLocal(callable, call, output, true)
)
}
predicate basicStoreStep(I::Node nodeFrom, I::Node nodeTo, I::TypeTrackerContent content) {
exists(
I::SummarizedCallable callable, I::Node call, I::SummaryComponentStack input,
I::SummaryComponentStack output
|
hasStoreSummary(callable, content, pragma[only_bind_into](input),
pragma[only_bind_into](output)) and
call = I::callTo(callable) and
nodeFrom = evaluateSummaryComponentStackLocal(callable, call, input, false) and
nodeTo = evaluateSummaryComponentStackLocal(callable, call, output, true)
)
}
predicate basicLoadStoreStep(
I::Node nodeFrom, I::Node nodeTo, I::TypeTrackerContent loadContent,
I::TypeTrackerContent storeContent
) {
exists(
I::SummarizedCallable callable, I::Node call, I::SummaryComponentStack input,
I::SummaryComponentStack output
|
hasLoadStoreSummary(callable, loadContent, storeContent, pragma[only_bind_into](input),
pragma[only_bind_into](output)) and
call = I::callTo(callable) and
nodeFrom = evaluateSummaryComponentStackLocal(callable, call, input, false) and
nodeTo = evaluateSummaryComponentStackLocal(callable, call, output, true)
)
}
predicate basicWithoutContentStep(
I::Node nodeFrom, I::Node nodeTo, I::TypeTrackerContentFilter filter
) {
exists(
I::SummarizedCallable callable, I::Node call, I::SummaryComponentStack input,
I::SummaryComponentStack output
|
hasWithoutContentSummary(callable, filter, pragma[only_bind_into](input),
pragma[only_bind_into](output)) and
call = I::callTo(callable) and
nodeFrom = evaluateSummaryComponentStackLocal(callable, call, input, false) and
nodeTo = evaluateSummaryComponentStackLocal(callable, call, output, true)
)
}
predicate basicWithContentStep(
I::Node nodeFrom, I::Node nodeTo, I::TypeTrackerContentFilter filter
) {
exists(
I::SummarizedCallable callable, I::Node call, I::SummaryComponentStack input,
I::SummaryComponentStack output
|
hasWithContentSummary(callable, filter, pragma[only_bind_into](input),
pragma[only_bind_into](output)) and
call = I::callTo(callable) and
nodeFrom = evaluateSummaryComponentStackLocal(callable, call, input, false) and
nodeTo = evaluateSummaryComponentStackLocal(callable, call, output, true)
)
}
}

108
python/ql/lib/semmle/python/dataflow/new/internal/TypeTracker.qll
View File

@@ -8,22 +8,22 @@ private module Cached {
* A description of a step on an inter-procedural data flow path.
*/
cached
newtype TStepSummary =
deprecated newtype TStepSummary =
LevelStep() or
CallStep() or
ReturnStep() or
StoreStep(TypeTrackerContent content) { basicStoreStep(_, _, content) } or
LoadStep(TypeTrackerContent content) { basicLoadStep(_, _, content) } or
LoadStoreStep(TypeTrackerContent load, TypeTrackerContent store) {
deprecated StoreStep(TypeTrackerContent content) { basicStoreStep(_, _, content) } or
deprecated LoadStep(TypeTrackerContent content) { basicLoadStep(_, _, content) } or
deprecated LoadStoreStep(TypeTrackerContent load, TypeTrackerContent store) {
basicLoadStoreStep(_, _, load, store)
} or
WithContent(ContentFilter filter) { basicWithContentStep(_, _, filter) } or
WithoutContent(ContentFilter filter) { basicWithoutContentStep(_, _, filter) } or
deprecated WithContent(ContentFilter filter) { basicWithContentStep(_, _, filter) } or
deprecated WithoutContent(ContentFilter filter) { basicWithoutContentStep(_, _, filter) } or
JumpStep()
cached
newtype TTypeTracker =
MkTypeTracker(Boolean hasCall, OptionalTypeTrackerContent content) {
deprecated newtype TTypeTracker =
deprecated MkTypeTracker(Boolean hasCall, OptionalTypeTrackerContent content) {
content = noContent()
or
// Restrict `content` to those that might eventually match a load.
@@ -40,8 +40,8 @@ private module Cached {
}
cached
newtype TTypeBackTracker =
MkTypeBackTracker(Boolean hasReturn, OptionalTypeTrackerContent content) {
deprecated newtype TTypeBackTracker =
deprecated MkTypeBackTracker(Boolean hasReturn, OptionalTypeTrackerContent content) {
content = noContent()
or
// As in MkTypeTracker, restrict `content` to those that might eventually match a store.
@@ -57,11 +57,13 @@ private module Cached {
/** Gets a type tracker with no content and the call bit set to the given value. */
cached
TypeTracker noContentTypeTracker(boolean hasCall) { result = MkTypeTracker(hasCall, noContent()) }
deprecated TypeTracker noContentTypeTracker(boolean hasCall) {
result = MkTypeTracker(hasCall, noContent())
}
/** Gets the summary resulting from appending `step` to type-tracking summary `tt`. */
cached
TypeTracker append(TypeTracker tt, StepSummary step) {
deprecated TypeTracker append(TypeTracker tt, StepSummary step) {
exists(Boolean hasCall, OptionalTypeTrackerContent currentContents |
tt = MkTypeTracker(hasCall, currentContents)
|
@@ -108,13 +110,13 @@ private module Cached {
}
pragma[nomagic]
private TypeBackTracker noContentTypeBackTracker(boolean hasReturn) {
deprecated private TypeBackTracker noContentTypeBackTracker(boolean hasReturn) {
result = MkTypeBackTracker(hasReturn, noContent())
}
/** Gets the summary resulting from prepending `step` to this type-tracking summary. */
cached
TypeBackTracker prepend(TypeBackTracker tbt, StepSummary step) {
deprecated TypeBackTracker prepend(TypeBackTracker tbt, StepSummary step) {
exists(Boolean hasReturn, OptionalTypeTrackerContent content |
tbt = MkTypeBackTracker(hasReturn, content)
|
@@ -167,7 +169,9 @@ private module Cached {
* Steps contained in this predicate should _not_ depend on the call graph.
*/
cached
predicate stepNoCall(TypeTrackingNode nodeFrom, TypeTrackingNode nodeTo, StepSummary summary) {
deprecated predicate stepNoCall(
TypeTrackingNode nodeFrom, TypeTrackingNode nodeTo, StepSummary summary
) {
exists(Node mid | nodeFrom.flowsTo(mid) and smallstepNoCall(mid, nodeTo, summary))
}
@@ -176,12 +180,14 @@ private module Cached {
* inter-procedural step from `nodeFrom` to `nodeTo`.
*/
cached
predicate stepCall(TypeTrackingNode nodeFrom, TypeTrackingNode nodeTo, StepSummary summary) {
deprecated predicate stepCall(
TypeTrackingNode nodeFrom, TypeTrackingNode nodeTo, StepSummary summary
) {
exists(Node mid | nodeFrom.flowsTo(mid) and smallstepCall(mid, nodeTo, summary))
}
cached
predicate smallstepNoCall(Node nodeFrom, TypeTrackingNode nodeTo, StepSummary summary) {
deprecated predicate smallstepNoCall(Node nodeFrom, TypeTrackingNode nodeTo, StepSummary summary) {
jumpStep(nodeFrom, nodeTo) and
summary = JumpStep()
or
@@ -210,7 +216,7 @@ private module Cached {
}
cached
predicate smallstepCall(Node nodeFrom, TypeTrackingNode nodeTo, StepSummary summary) {
deprecated predicate smallstepCall(Node nodeFrom, TypeTrackingNode nodeTo, StepSummary summary) {
callStep(nodeFrom, nodeTo) and summary = CallStep()
or
returnStep(nodeFrom, nodeTo) and
@@ -223,25 +229,27 @@ private module Cached {
private import Cached
private predicate step(TypeTrackingNode nodeFrom, TypeTrackingNode nodeTo, StepSummary summary) {
deprecated private predicate step(
TypeTrackingNode nodeFrom, TypeTrackingNode nodeTo, StepSummary summary
) {
stepNoCall(nodeFrom, nodeTo, summary)
or
stepCall(nodeFrom, nodeTo, summary)
}
pragma[nomagic]
private predicate stepProj(TypeTrackingNode nodeFrom, StepSummary summary) {
deprecated private predicate stepProj(TypeTrackingNode nodeFrom, StepSummary summary) {
step(nodeFrom, _, summary)
}
private predicate smallstep(Node nodeFrom, TypeTrackingNode nodeTo, StepSummary summary) {
deprecated private predicate smallstep(Node nodeFrom, TypeTrackingNode nodeTo, StepSummary summary) {
smallstepNoCall(nodeFrom, nodeTo, summary)
or
smallstepCall(nodeFrom, nodeTo, summary)
}
pragma[nomagic]
private predicate smallstepProj(Node nodeFrom, StepSummary summary) {
deprecated private predicate smallstepProj(Node nodeFrom, StepSummary summary) {
smallstep(nodeFrom, _, summary)
}
@@ -270,7 +278,7 @@ private predicate smallstepProj(Node nodeFrom, StepSummary summary) {
* function. This means we will track the fact that `x.attr` can have the type of `y` into the
* assignment to `z` inside `bar`, even though this attribute write happens _after_ `bar` is called.
*/
private predicate flowsToStoreStep(
deprecated private predicate flowsToStoreStep(
Node nodeFrom, TypeTrackingNode nodeTo, TypeTrackerContent content
) {
exists(Node obj | nodeTo.flowsTo(obj) and basicStoreStep(nodeFrom, obj, content))
@@ -279,7 +287,7 @@ private predicate flowsToStoreStep(
/**
* Holds if `loadContent` is loaded from `nodeFrom` and written to `storeContent` of `nodeTo`.
*/
private predicate flowsToLoadStoreStep(
deprecated private predicate flowsToLoadStoreStep(
Node nodeFrom, TypeTrackingNode nodeTo, TypeTrackerContent loadContent,
TypeTrackerContent storeContent
) {
@@ -293,7 +301,7 @@ private predicate flowsToLoadStoreStep(
*
* A description of a step on an inter-procedural data flow path.
*/
class StepSummary extends TStepSummary {
deprecated class StepSummary extends TStepSummary {
/** Gets a textual representation of this step summary. */
string toString() {
this instanceof LevelStep and result = "level"
@@ -316,7 +324,7 @@ class StepSummary extends TStepSummary {
}
/** Provides predicates for updating step summaries (`StepSummary`s). */
module StepSummary {
deprecated module StepSummary {
predicate append = Cached::append/2;
/**
@@ -411,6 +419,8 @@ module StepSummary {
}
/**
* DEPRECATED: Use `semmle.python.dataflow.new.TypeTracking` instead.
*
* A summary of the steps needed to track a value to a given dataflow node.
*
* This can be used to track objects that implement a certain API in order to
@@ -437,7 +447,7 @@ module StepSummary {
* `t = t2.step(myType(t2), result)`. If you additionally want to track individual
* intra-procedural steps, use `t = t2.smallstep(myCallback(t2), result)`.
*/
class TypeTracker extends TTypeTracker {
deprecated class TypeTracker extends TTypeTracker {
Boolean hasCall;
OptionalTypeTrackerContent content;
@@ -565,7 +575,7 @@ class TypeTracker extends TTypeTracker {
}
/** Provides predicates for implementing custom `TypeTracker`s. */
module TypeTracker {
deprecated module TypeTracker {
/**
* Gets a valid end point of type tracking.
*/
@@ -580,15 +590,17 @@ module TypeTracker {
}
pragma[nomagic]
private predicate backStepProj(TypeTrackingNode nodeTo, StepSummary summary) {
deprecated private predicate backStepProj(TypeTrackingNode nodeTo, StepSummary summary) {
step(_, nodeTo, summary)
}
private predicate backSmallstepProj(TypeTrackingNode nodeTo, StepSummary summary) {
deprecated private predicate backSmallstepProj(TypeTrackingNode nodeTo, StepSummary summary) {
smallstep(_, nodeTo, summary)
}
/**
* DEPRECATED: Use `semmle.python.dataflow.new.TypeTracking` instead.
*
* A summary of the steps needed to back-track a use of a value to a given dataflow node.
*
* This can for example be used to track callbacks that are passed to a certain API,
@@ -618,7 +630,7 @@ private predicate backSmallstepProj(TypeTrackingNode nodeTo, StepSummary summary
* `t2 = t.step(result, myCallback(t2))`. If you additionally want to track individual
* intra-procedural steps, use `t2 = t.smallstep(result, myCallback(t2))`.
*/
class TypeBackTracker extends TTypeBackTracker {
deprecated class TypeBackTracker extends TTypeBackTracker {
Boolean hasReturn;
OptionalTypeTrackerContent content;
@@ -747,7 +759,7 @@ class TypeBackTracker extends TTypeBackTracker {
}
/** Provides predicates for implementing custom `TypeBackTracker`s. */
module TypeBackTracker {
deprecated module TypeBackTracker {
/**
* Gets a valid end point of type back-tracking.
*/
@@ -768,14 +780,14 @@ module TypeBackTracker {
* `stepCall` relation (`stepNoCall` not being recursive, can be join-ordered in the
* same way as in `stepInlineLate`).
*/
module CallGraphConstruction {
deprecated module CallGraphConstruction {
/** The input to call graph construction. */
signature module InputSig {
/** A state to track during type tracking. */
class State;
/** Holds if type tracking should start at `start` in state `state`. */
predicate start(Node start, State state);
deprecated predicate start(Node start, State state);
/**
* Holds if type tracking should use the step from `nodeFrom` to `nodeTo`,
@@ -784,7 +796,7 @@ module CallGraphConstruction {
* Implementing this predicate using `StepSummary::[small]stepNoCall` yields
* standard type tracking.
*/
predicate stepNoCall(Node nodeFrom, Node nodeTo, StepSummary summary);
deprecated predicate stepNoCall(Node nodeFrom, Node nodeTo, StepSummary summary);
/**
* Holds if type tracking should use the step from `nodeFrom` to `nodeTo`,
@@ -793,7 +805,7 @@ module CallGraphConstruction {
* Implementing this predicate using `StepSummary::[small]stepCall` yields
* standard type tracking.
*/
predicate stepCall(Node nodeFrom, Node nodeTo, StepSummary summary);
deprecated predicate stepCall(Node nodeFrom, Node nodeTo, StepSummary summary);
/** A projection of an element from the state space. */
class StateProj;
@@ -802,25 +814,25 @@ module CallGraphConstruction {
StateProj stateProj(State state);
/** Holds if type tracking should stop at `n` when we are tracking projected state `stateProj`. */
predicate filter(Node n, StateProj stateProj);
deprecated predicate filter(Node n, StateProj stateProj);
}
/** Provides the `track` predicate for use in call graph construction. */
module Make<InputSig Input> {
pragma[nomagic]
private predicate stepNoCallProj(Node nodeFrom, StepSummary summary) {
deprecated private predicate stepNoCallProj(Node nodeFrom, StepSummary summary) {
Input::stepNoCall(nodeFrom, _, summary)
}
pragma[nomagic]
private predicate stepCallProj(Node nodeFrom, StepSummary summary) {
deprecated private predicate stepCallProj(Node nodeFrom, StepSummary summary) {
Input::stepCall(nodeFrom, _, summary)
}
bindingset[nodeFrom, t]
pragma[inline_late]
pragma[noopt]
private TypeTracker stepNoCallInlineLate(
deprecated private TypeTracker stepNoCallInlineLate(
TypeTracker t, TypeTrackingNode nodeFrom, TypeTrackingNode nodeTo
) {
exists(StepSummary summary |
@@ -837,7 +849,7 @@ module CallGraphConstruction {
}
pragma[nomagic]
private Node track(Input::State state, TypeTracker t) {
deprecated private Node track(Input::State state, TypeTracker t) {
t.start() and Input::start(result, state)
or
exists(Input::StateProj stateProj |
@@ -855,12 +867,12 @@ module CallGraphConstruction {
bindingset[t, summary]
pragma[inline_late]
private TypeTracker appendInlineLate(TypeTracker t, StepSummary summary) {
deprecated private TypeTracker appendInlineLate(TypeTracker t, StepSummary summary) {
result = t.append(summary)
}
pragma[nomagic]
private Node trackCall(Input::State state, TypeTracker t, StepSummary summary) {
deprecated private Node trackCall(Input::State state, TypeTracker t, StepSummary summary) {
exists(TypeTracker t2 |
// non-linear recursion
result = track(state, t2) and
@@ -871,7 +883,7 @@ module CallGraphConstruction {
/** Gets a node that can be reached from _some_ start node in state `state`. */
pragma[nomagic]
Node track(Input::State state) { result = track(state, TypeTracker::end()) }
deprecated Node track(Input::State state) { result = track(state, TypeTracker::end()) }
}
/** A simple version of `CallGraphConstruction` that uses standard type tracking. */
@@ -882,15 +894,15 @@ module CallGraphConstruction {
class State;
/** Holds if type tracking should start at `start` in state `state`. */
predicate start(Node start, State state);
deprecated predicate start(Node start, State state);
/** Holds if type tracking should stop at `n`. */
predicate filter(Node n);
deprecated predicate filter(Node n);
}
/** Provides the `track` predicate for use in call graph construction. */
module Make<InputSig Input> {
private module I implements CallGraphConstruction::InputSig {
deprecated private module I implements CallGraphConstruction::InputSig {
private import codeql.util.Unit
class State = Input::State;
@@ -915,7 +927,7 @@ module CallGraphConstruction {
}
}
import CallGraphConstruction::Make<I>
deprecated import CallGraphConstruction::Make<I>
}
}
}

223
python/ql/lib/semmle/python/dataflow/new/internal/TypeTrackerSpecific.qll
View File

@@ -4,87 +4,54 @@
private import python
private import semmle.python.dataflow.new.internal.DataFlowPublic as DataFlowPublic
private import semmle.python.dataflow.new.internal.DataFlowPrivate as DataFlowPrivate
import semmle.python.internal.CachedStages
private import TypeTrackingImpl as TypeTrackingImpl
class Node = DataFlowPublic::Node;
deprecated class Node = DataFlowPublic::Node;
class TypeTrackingNode = DataFlowPublic::TypeTrackingNode;
deprecated class TypeTrackingNode = DataFlowPublic::TypeTrackingNode;
/** A content name for use by type trackers, or the empty string. */
class OptionalTypeTrackerContent extends string {
deprecated class OptionalTypeTrackerContent extends string {
OptionalTypeTrackerContent() {
this = ""
or
this = getPossibleContentName()
this instanceof TypeTrackingImpl::TypeTrackingInput::Content
}
}
/** A content name for use by type trackers. */
class TypeTrackerContent extends OptionalTypeTrackerContent {
deprecated class TypeTrackerContent extends OptionalTypeTrackerContent {
TypeTrackerContent() { this != "" }
}
/** Gets the content string representing no value. */
OptionalTypeTrackerContent noContent() { result = "" }
deprecated OptionalTypeTrackerContent noContent() { result = "" }
/**
* A label to use for `WithContent` and `WithoutContent` steps, restricting
* which `ContentSet` may pass through. Not currently used in Python.
*/
class ContentFilter extends Unit {
deprecated class ContentFilter extends Unit {
TypeTrackerContent getAMatchingContent() { none() }
}
pragma[inline]
predicate compatibleContents(TypeTrackerContent storeContent, TypeTrackerContent loadContent) {
deprecated predicate compatibleContents(
TypeTrackerContent storeContent, TypeTrackerContent loadContent
) {
storeContent = loadContent
}
predicate simpleLocalFlowStep = DataFlowPrivate::simpleLocalFlowStepForTypetracking/2;
deprecated predicate simpleLocalFlowStep =
TypeTrackingImpl::TypeTrackingInput::simpleLocalSmallStep/2;
predicate jumpStep(Node nodeFrom, Node nodeTo) {
DataFlowPrivate::jumpStepSharedWithTypeTracker(nodeFrom, nodeTo)
or
capturedJumpStep(nodeFrom, nodeTo)
}
predicate capturedJumpStep(Node nodeFrom, Node nodeTo) {
// Jump into a capturing scope.
//
// var = expr
// ...
// def f():
// ..var is used..
//
// nodeFrom is `expr`
// nodeTo is entry node for `f`
exists(ScopeEntryDefinition e, SsaSourceVariable var, DefinitionNode def |
e.getSourceVariable() = var and
var.hasDefiningNode(def)
|
nodeTo.asCfgNode() = e.getDefiningNode() and
nodeFrom.asCfgNode() = def.getValue() and
var.getScope().getScope*() = nodeFrom.getScope()
)
}
deprecated predicate jumpStep = TypeTrackingImpl::TypeTrackingInput::jumpStep/2;
/** Holds if there is a level step from `nodeFrom` to `nodeTo`, which may depend on the call graph. */
predicate levelStepCall(Node nodeFrom, Node nodeTo) { none() }
deprecated predicate levelStepCall(Node nodeFrom, Node nodeTo) { none() }
/** Holds if there is a level step from `nodeFrom` to `nodeTo`, which does not depend on the call graph. */
predicate levelStepNoCall(Node nodeFrom, Node nodeTo) {
TypeTrackerSummaryFlow::levelStepNoCall(nodeFrom, nodeTo)
}
/**
* Gets the name of a possible piece of content. For Python, this is currently only attribute names,
* using the name of the attribute for the corresponding content.
*/
string getPossibleContentName() {
Stages::TypeTracking::ref() and // the TypeTracking::append() etc. predicates that we want to cache depend on this predicate, so we can place the `ref()` call here to get around identical files.
result = any(DataFlowPublic::AttrRef a).getAttributeName()
}
deprecated predicate levelStepNoCall = TypeTrackingImpl::TypeTrackingInput::levelStepNoCall/2;
/**
* Holds if `nodeFrom` steps to `nodeTo` by being passed as a parameter in a call.
@@ -93,177 +60,43 @@ string getPossibleContentName() {
* recursion (or, at best, terrible performance), since identifying calls to library
* methods is done using API graphs (which uses type tracking).
*/
predicate callStep(DataFlowPublic::ArgumentNode nodeFrom, DataFlowPublic::ParameterNode nodeTo) {
exists(
DataFlowPrivate::DataFlowCall call, DataFlowPrivate::DataFlowCallable callable,
DataFlowPrivate::ArgumentPosition apos, DataFlowPrivate::ParameterPosition ppos
|
nodeFrom = call.getArgument(apos) and
nodeTo = callable.getParameter(ppos) and
DataFlowPrivate::parameterMatch(ppos, apos) and
callable = call.getCallable()
)
}
deprecated predicate callStep = TypeTrackingImpl::TypeTrackingInput::callStep/2;
/** Holds if `nodeFrom` steps to `nodeTo` by being returned from a call. */
predicate returnStep(DataFlowPrivate::ReturnNode nodeFrom, Node nodeTo) {
exists(DataFlowPrivate::ExtractedDataFlowCall call |
nodeFrom.getEnclosingCallable() = call.getCallable() and
nodeTo.(DataFlowPublic::CfgNode).getNode() = call.getNode()
)
}
deprecated predicate returnStep = TypeTrackingImpl::TypeTrackingInput::returnStep/2;
/**
* Holds if `nodeFrom` is being written to the `content` content of the object in `nodeTo`.
*/
predicate basicStoreStep(Node nodeFrom, Node nodeTo, string content) {
exists(DataFlowPublic::AttrWrite a |
a.mayHaveAttributeName(content) and
nodeFrom = a.getValue() and
nodeTo = a.getObject()
)
or
exists(DataFlowPublic::ContentSet contents |
contents.(DataFlowPublic::AttributeContent).getAttribute() = content
|
TypeTrackerSummaryFlow::basicStoreStep(nodeFrom, nodeTo, contents)
)
}
deprecated predicate basicStoreStep = TypeTrackingImpl::TypeTrackingInput::storeStep/3;
/**
* Holds if `nodeTo` is the result of accessing the `content` content of `nodeFrom`.
*/
predicate basicLoadStep(Node nodeFrom, Node nodeTo, string content) {
exists(DataFlowPublic::AttrRead a |
a.mayHaveAttributeName(content) and
nodeFrom = a.getObject() and
nodeTo = a
)
or
exists(DataFlowPublic::ContentSet contents |
contents.(DataFlowPublic::AttributeContent).getAttribute() = content
|
TypeTrackerSummaryFlow::basicLoadStep(nodeFrom, nodeTo, contents)
)
}
deprecated predicate basicLoadStep = TypeTrackingImpl::TypeTrackingInput::loadStep/3;
/**
* Holds if the `loadContent` of `nodeFrom` is stored in the `storeContent` of `nodeTo`.
*/
predicate basicLoadStoreStep(Node nodeFrom, Node nodeTo, string loadContent, string storeContent) {
exists(DataFlowPublic::ContentSet loadContents, DataFlowPublic::ContentSet storeContents |
loadContents.(DataFlowPublic::AttributeContent).getAttribute() = loadContent and
storeContents.(DataFlowPublic::AttributeContent).getAttribute() = storeContent
|
TypeTrackerSummaryFlow::basicLoadStoreStep(nodeFrom, nodeTo, loadContents, storeContents)
)
}
deprecated predicate basicLoadStoreStep = TypeTrackingImpl::TypeTrackingInput::loadStoreStep/4;
/**
* Holds if type-tracking should step from `nodeFrom` to `nodeTo` but block flow of contents matched by `filter` through here.
*/
predicate basicWithoutContentStep(Node nodeFrom, Node nodeTo, ContentFilter filter) { none() }
deprecated predicate basicWithoutContentStep(Node nodeFrom, Node nodeTo, ContentFilter filter) {
none()
}
/**
* Holds if type-tracking should step from `nodeFrom` to `nodeTo` if inside a content matched by `filter`.
*/
predicate basicWithContentStep(Node nodeFrom, Node nodeTo, ContentFilter filter) { none() }
deprecated predicate basicWithContentStep(Node nodeFrom, Node nodeTo, ContentFilter filter) {
none()
}
/**
* A utility class that is equivalent to `boolean` but does not require type joining.
*/
class Boolean extends boolean {
deprecated class Boolean extends boolean {
Boolean() { this = true or this = false }
}
private import SummaryTypeTracker as SummaryTypeTracker
private import semmle.python.dataflow.new.internal.FlowSummaryImpl as FlowSummaryImpl
private import semmle.python.dataflow.new.internal.DataFlowDispatch as DataFlowDispatch
pragma[noinline]
private predicate argumentPositionMatch(
DataFlowPublic::CallCfgNode call, DataFlowPublic::Node arg,
DataFlowDispatch::ParameterPosition ppos
) {
exists(DataFlowDispatch::ArgumentPosition apos |
DataFlowDispatch::parameterMatch(ppos, apos) and
DataFlowDispatch::normalCallArg(call.getNode(), arg, apos)
)
}
private module SummaryTypeTrackerInput implements SummaryTypeTracker::Input {
// Dataflow nodes
class Node = DataFlowPublic::Node;
// Content
class TypeTrackerContent = DataFlowPublic::ContentSet;
class TypeTrackerContentFilter = ContentFilter;
TypeTrackerContentFilter getFilterFromWithoutContentStep(TypeTrackerContent content) { none() }
TypeTrackerContentFilter getFilterFromWithContentStep(TypeTrackerContent content) { none() }
// Callables
class SummarizedCallable = FlowSummaryImpl::Private::SummarizedCallableImpl;
// Summaries and their stacks
class SummaryComponent = FlowSummaryImpl::Private::SummaryComponent;
class SummaryComponentStack = FlowSummaryImpl::Private::SummaryComponentStack;
predicate singleton = FlowSummaryImpl::Private::SummaryComponentStack::singleton/1;
predicate push = FlowSummaryImpl::Private::SummaryComponentStack::push/2;
// Relating content to summaries
predicate content = FlowSummaryImpl::Private::SummaryComponent::content/1;
SummaryComponent withoutContent(TypeTrackerContent contents) { none() }
SummaryComponent withContent(TypeTrackerContent contents) { none() }
predicate return = FlowSummaryImpl::Private::SummaryComponent::return/0;
// Relating nodes to summaries
Node argumentOf(Node call, SummaryComponent arg, boolean isPostUpdate) {
exists(DataFlowDispatch::ParameterPosition pos |
arg = FlowSummaryImpl::Private::SummaryComponent::argument(pos) and
argumentPositionMatch(call, result, pos) and
isPostUpdate = [false, true] // todo: implement when/if Python uses post-update nodes in type tracking
)
}
Node parameterOf(Node callable, SummaryComponent param) {
exists(
DataFlowDispatch::ArgumentPosition apos, DataFlowDispatch::ParameterPosition ppos, Parameter p
|
param = FlowSummaryImpl::Private::SummaryComponent::parameter(apos) and
DataFlowDispatch::parameterMatch(ppos, apos) and
result.asCfgNode().getNode() = p and
(
exists(int i | ppos.isPositional(i) |
p = callable.getALocalSource().asExpr().(CallableExpr).getInnerScope().getArg(i)
)
or
exists(string name | ppos.isKeyword(name) |
p = callable.getALocalSource().asExpr().(CallableExpr).getInnerScope().getArgByName(name)
)
)
)
}
Node returnOf(Node callable, SummaryComponent return) {
return = FlowSummaryImpl::Private::SummaryComponent::return() and
// `result` should be the return value of a callable expression (lambda or function) referenced by `callable`
result.asCfgNode() =
callable.getALocalSource().asExpr().(CallableExpr).getInnerScope().getAReturnValueFlowNode()
}
// Relating callables to nodes
Node callTo(SummarizedCallable callable) {
result = callable.(DataFlowDispatch::LibraryCallable).getACallSimple()
}
}
private module TypeTrackerSummaryFlow = SummaryTypeTracker::SummaryFlow<SummaryTypeTrackerInput>;

274
python/ql/lib/semmle/python/dataflow/new/internal/TypeTrackingImpl.qll Normal file
View File

@@ -0,0 +1,274 @@
import codeql.util.Unit
import codeql.typetracking.TypeTracking as Shared
import codeql.typetracking.internal.TypeTrackingImpl as SharedImpl
private import python
private import semmle.python.internal.CachedStages
private import semmle.python.dataflow.new.internal.DataFlowPublic as DataFlowPublic
private import semmle.python.dataflow.new.internal.DataFlowPrivate as DataFlowPrivate
private import codeql.typetracking.internal.SummaryTypeTracker as SummaryTypeTracker
private import semmle.python.dataflow.new.internal.FlowSummaryImpl as FlowSummaryImpl
private import semmle.python.dataflow.new.internal.DataFlowDispatch as DataFlowDispatch
private module SummaryTypeTrackerInput implements SummaryTypeTracker::Input {
// Dataflow nodes
class Node = DataFlowPublic::Node;
// Content
class Content = DataFlowPublic::ContentSet;
class ContentFilter = TypeTrackingInput::ContentFilter;
ContentFilter getFilterFromWithoutContentStep(Content content) { none() }
ContentFilter getFilterFromWithContentStep(Content content) { none() }
// Callables
class SummarizedCallable = FlowSummaryImpl::Private::SummarizedCallableImpl;
// Summaries and their stacks
class SummaryComponent = FlowSummaryImpl::Private::SummaryComponent;
class SummaryComponentStack = FlowSummaryImpl::Private::SummaryComponentStack;
predicate singleton = FlowSummaryImpl::Private::SummaryComponentStack::singleton/1;
predicate push = FlowSummaryImpl::Private::SummaryComponentStack::push/2;
// Relating content to summaries
predicate content = FlowSummaryImpl::Private::SummaryComponent::content/1;
SummaryComponent withoutContent(Content contents) { none() }
SummaryComponent withContent(Content contents) { none() }
predicate return = FlowSummaryImpl::Private::SummaryComponent::return/0;
pragma[noinline]
private predicate argumentPositionMatch(
DataFlowPublic::CallCfgNode call, DataFlowPublic::Node arg,
DataFlowDispatch::ParameterPosition ppos
) {
exists(DataFlowDispatch::ArgumentPosition apos |
DataFlowDispatch::parameterMatch(ppos, apos) and
DataFlowDispatch::normalCallArg(call.getNode(), arg, apos)
)
}
// Relating nodes to summaries
Node argumentOf(Node call, SummaryComponent arg, boolean isPostUpdate) {
exists(DataFlowDispatch::ParameterPosition pos |
arg = FlowSummaryImpl::Private::SummaryComponent::argument(pos) and
argumentPositionMatch(call, result, pos) and
isPostUpdate = [false, true] // todo: implement when/if Python uses post-update nodes in type tracking
)
}
Node parameterOf(Node callable, SummaryComponent param) {
exists(
DataFlowDispatch::ArgumentPosition apos, DataFlowDispatch::ParameterPosition ppos, Parameter p
|
param = FlowSummaryImpl::Private::SummaryComponent::parameter(apos) and
DataFlowDispatch::parameterMatch(ppos, apos) and
result.asCfgNode().getNode() = p and
(
exists(int i | ppos.isPositional(i) |
p = callable.getALocalSource().asExpr().(CallableExpr).getInnerScope().getArg(i)
)
or
exists(string name | ppos.isKeyword(name) |
p = callable.getALocalSource().asExpr().(CallableExpr).getInnerScope().getArgByName(name)
)
)
)
}
Node returnOf(Node callable, SummaryComponent return) {
return = FlowSummaryImpl::Private::SummaryComponent::return() and
// `result` should be the return value of a callable expression (lambda or function) referenced by `callable`
result.asCfgNode() =
callable.getALocalSource().asExpr().(CallableExpr).getInnerScope().getAReturnValueFlowNode()
}
// Relating callables to nodes
Node callTo(SummarizedCallable callable) {
result = callable.(DataFlowDispatch::LibraryCallable).getACallSimple()
}
}
private module TypeTrackerSummaryFlow = SummaryTypeTracker::SummaryFlow<SummaryTypeTrackerInput>;
/**
* Gets the name of a possible piece of content. For Python, this is currently only attribute names,
* using the name of the attribute for the corresponding content.
*/
private string getPossibleContentName() {
Stages::TypeTracking::ref() and // the TypeTracking::append() etc. predicates that we want to cache depend on this predicate, so we can place the `ref()` call here to get around identical files.
result = any(DataFlowPublic::AttrRef a).getAttributeName()
}
module TypeTrackingInput implements Shared::TypeTrackingInput {
class Node = DataFlowPublic::Node;
class LocalSourceNode = DataFlowPublic::LocalSourceNode;
class Content instanceof string {
Content() { this = getPossibleContentName() }
string toString() { result = this }
}
/**
* A label to use for `WithContent` and `WithoutContent` steps, restricting
* which `ContentSet` may pass through.
*/
class ContentFilter extends Unit {
Content getAMatchingContent() { none() }
}
/**
* Holds if a value stored with `storeContents` can be read back with `loadContents`.
*/
pragma[inline]
predicate compatibleContents(Content storeContents, Content loadContents) {
storeContents = loadContents
}
/** Holds if there is a simple local flow step from `nodeFrom` to `nodeTo` */
predicate simpleLocalSmallStep = DataFlowPrivate::simpleLocalFlowStepForTypetracking/2;
/** Holds if there is a level step from `nodeFrom` to `nodeTo`, which may depend on the call graph. */
predicate levelStepCall(Node nodeFrom, LocalSourceNode nodeTo) { none() }
/** Holds if there is a level step from `nodeFrom` to `nodeTo`, which does not depend on the call graph. */
predicate levelStepNoCall(Node nodeFrom, LocalSourceNode nodeTo) {
TypeTrackerSummaryFlow::levelStepNoCall(nodeFrom, nodeTo)
}
/**
* Holds if `nodeFrom` steps to `nodeTo` by being passed as a parameter in a call.
*
* Flow into summarized library methods is not included, as that will lead to negative
* recursion (or, at best, terrible performance), since identifying calls to library
* methods is done using API graphs (which uses type tracking).
*/
predicate callStep(Node nodeFrom, LocalSourceNode nodeTo) {
exists(
DataFlowPrivate::DataFlowCall call, DataFlowPrivate::DataFlowCallable callable,
DataFlowPrivate::ArgumentPosition apos, DataFlowPrivate::ParameterPosition ppos
|
nodeFrom = call.getArgument(apos) and
nodeTo = callable.getParameter(ppos) and
DataFlowPrivate::parameterMatch(ppos, apos) and
callable = call.getCallable()
)
}
/**
* Holds if `nodeFrom` steps to `nodeTo` by being returned from a call.
*
* Flow out of summarized library methods is not included, as that will lead to negative
* recursion (or, at best, terrible performance), since identifying calls to library
* methods is done using API graphs (which uses type tracking).
*/
predicate returnStep(Node nodeFrom, LocalSourceNode nodeTo) {
exists(DataFlowPrivate::ExtractedDataFlowCall call |
nodeFrom.(DataFlowPrivate::ReturnNode).getEnclosingCallable() = call.getCallable() and
nodeTo.(DataFlowPublic::CfgNode).getNode() = call.getNode()
)
}
/**
* Holds if `nodeFrom` is being written to the `content` content of the object in `nodeTo`.
*/
predicate storeStep(Node nodeFrom, Node nodeTo, Content content) {
exists(DataFlowPublic::AttrWrite a |
a.mayHaveAttributeName(content) and
nodeFrom = a.getValue() and
nodeTo = a.getObject()
)
or
exists(DataFlowPublic::ContentSet contents |
contents.(DataFlowPublic::AttributeContent).getAttribute() = content
|
TypeTrackerSummaryFlow::basicStoreStep(nodeFrom, nodeTo, contents)
)
}
/**
* Holds if `nodeTo` is the result of accessing the `content` content of `nodeFrom`.
*/
predicate loadStep(Node nodeFrom, LocalSourceNode nodeTo, Content content) {
exists(DataFlowPublic::AttrRead a |
a.mayHaveAttributeName(content) and
nodeFrom = a.getObject() and
nodeTo = a
)
or
exists(DataFlowPublic::ContentSet contents |
contents.(DataFlowPublic::AttributeContent).getAttribute() = content
|
TypeTrackerSummaryFlow::basicLoadStep(nodeFrom, nodeTo, contents)
)
}
/**
* Holds if the `loadContent` of `nodeFrom` is stored in the `storeContent` of `nodeTo`.
*/
predicate loadStoreStep(Node nodeFrom, Node nodeTo, Content loadContent, Content storeContent) {
exists(DataFlowPublic::ContentSet loadContents, DataFlowPublic::ContentSet storeContents |
loadContents.(DataFlowPublic::AttributeContent).getAttribute() = loadContent and
storeContents.(DataFlowPublic::AttributeContent).getAttribute() = storeContent
|
TypeTrackerSummaryFlow::basicLoadStoreStep(nodeFrom, nodeTo, loadContents, storeContents)
)
}
/**
* Holds if type-tracking should step from `nodeFrom` to `nodeTo` if inside a content matched by `filter`.
*/
predicate withContentStep(Node nodeFrom, LocalSourceNode nodeTo, ContentFilter filter) {
TypeTrackerSummaryFlow::basicWithContentStep(nodeFrom, nodeTo, filter)
}
/**
* Holds if type-tracking should step from `nodeFrom` to `nodeTo` but block flow of contents matched by `filter` through here.
*/
predicate withoutContentStep(Node nodeFrom, LocalSourceNode nodeTo, ContentFilter filter) {
TypeTrackerSummaryFlow::basicWithoutContentStep(nodeFrom, nodeTo, filter)
}
private predicate capturedJumpStep(Node nodeFrom, Node nodeTo) {
// Jump into a capturing scope.
//
// var = expr
// ...
// def f():
// ..var is used..
//
// nodeFrom is `expr`
// nodeTo is entry node for `f`
exists(ScopeEntryDefinition e, SsaSourceVariable var, DefinitionNode def |
e.getSourceVariable() = var and
var.hasDefiningNode(def)
|
nodeTo.asCfgNode() = e.getDefiningNode() and
nodeFrom.asCfgNode() = def.getValue() and
var.getScope().getScope*() = nodeFrom.getScope()
)
}
/**
* Holds if data can flow from `node1` to `node2` in a way that discards call contexts.
*/
predicate jumpStep(Node nodeFrom, LocalSourceNode nodeTo) {
DataFlowPrivate::jumpStepSharedWithTypeTracker(nodeFrom, nodeTo)
or
capturedJumpStep(nodeFrom, nodeTo)
}
predicate hasFeatureBacktrackStoreTarget() { any() }
predicate nonStandardFlowsTo(LocalSourceNode localSource, Node dst) { localSource.flowsTo(dst) }
}
import SharedImpl::TypeTracking<TypeTrackingInput>

3
python/ql/lib/semmle/python/internal/CachedStages.qll
View File

@@ -111,6 +111,7 @@ module Stages {
predicate ref() { 1 = 1 }
private import semmle.python.dataflow.new.DataFlow::DataFlow as NewDataFlow
private import semmle.python.dataflow.new.internal.TypeTrackingImpl as TypeTrackingImpl
private import semmle.python.ApiGraphs::API as API
/**
@@ -121,7 +122,7 @@ module Stages {
predicate backref() {
1 = 1
or
exists(any(NewDataFlow::TypeTracker t).append(_))
exists(TypeTrackingImpl::append(_, _))
or
exists(any(API::Node n).getAMember().getAValueReachableFromSource())
}

2
python/ql/lib/semmle/python/security/dataflow/UnsafeShellCommandConstructionCustomizations.qll
View File

@@ -50,7 +50,7 @@ module UnsafeShellCommandConstruction {
source = backtrackShellExec(TypeTracker::TypeBackTracker::end(), shellExec)
}
import semmle.python.dataflow.new.TypeTracker as TypeTracker
import semmle.python.dataflow.new.TypeTracking as TypeTracker
private DataFlow::LocalSourceNode backtrackShellExec(
TypeTracker::TypeBackTracker t, Concepts::SystemCommandExecution shellExec

2
python/ql/test/experimental/dataflow/summaries/TestSummaries.qll
View File

@@ -8,7 +8,7 @@ private import semmle.python.ApiGraphs
* `getACall` predicate on `SummarizedCallable`.
*/
module RecursionGuard {
private import semmle.python.dataflow.new.internal.TypeTrackerSpecific as TT
private import semmle.python.dataflow.new.internal.TypeTrackingImpl::TypeTrackingInput as TT
private class RecursionGuard extends SummarizedCallable {
RecursionGuard() { this = "RecursionGuard" }

2
python/ql/test/experimental/dataflow/typetracking-summaries/TestSummaries.qll
View File

@@ -8,7 +8,7 @@ private import semmle.python.ApiGraphs
* `getACall` predicate on `SummarizedCallable`.
*/
module RecursionGuard {
private import semmle.python.dataflow.new.internal.TypeTrackerSpecific as TT
private import semmle.python.dataflow.new.internal.TypeTrackingImpl::TypeTrackingInput as TT
private class RecursionGuard extends SummarizedCallable {
RecursionGuard() { this = "TypeTrackingSummariesRecursionGuard" }

2
python/ql/test/experimental/dataflow/typetracking-summaries/tracked.ql
View File

@@ -1,6 +1,6 @@
import python
import semmle.python.dataflow.new.DataFlow
import semmle.python.dataflow.new.TypeTracker
import semmle.python.dataflow.new.TypeTracking
import TestUtilities.InlineExpectationsTest
import semmle.python.ApiGraphs
import TestSummaries

2
python/ql/test/experimental/dataflow/typetracking/moduleattr.ql
View File

@@ -1,6 +1,6 @@
import python
import semmle.python.dataflow.new.DataFlow
import semmle.python.dataflow.new.TypeTracker
import semmle.python.dataflow.new.TypeTracking
import semmle.python.ApiGraphs
private DataFlow::TypeTrackingNode module_tracker(TypeTracker t) {

2
python/ql/test/experimental/dataflow/typetracking/tracked.ql
View File

@@ -1,6 +1,6 @@
import python
import semmle.python.dataflow.new.DataFlow
import semmle.python.dataflow.new.TypeTracker
import semmle.python.dataflow.new.TypeTracking
import TestUtilities.InlineExpectationsTest
import semmle.python.ApiGraphs