Python: Adopt shared type tracking library

This commit is contained in:
Tom Hvitved
2023-11-20 13:16:57 +01:00
parent 4776e9ccd2
commit 3b1146bf98
17 changed files with 446 additions and 263 deletions

View File

@@ -4,9 +4,9 @@
*/
import python
private import semmle.python.dataflow.new.internal.TypeTrackerSpecific
private import semmle.python.dataflow.new.internal.TypeTrackingImpl
private import semmle.python.ApiGraphs
class CallCfgNodeWithTarget extends DataFlow::Node instanceof DataFlow::CallCfgNode {
DataFlow::Node getTarget() { returnStep(result, this) }
DataFlow::Node getTarget() { TypeTrackingInput::returnStep(result, this) }
}

View File

@@ -1,4 +1,6 @@
/**
* DEPRECATED: Use `semmle.python.dataflow.new.TypeTracking` instead.
*
* This file acts as a wrapper for `internal.TypeTracker`, exposing some of the functionality with
* names that are more appropriate for Python.
*/
@@ -8,12 +10,14 @@ private import internal.TypeTracker as Internal
private import internal.TypeTrackerSpecific as InternalSpecific
/** A string that may appear as the name of an attribute or access path. */
class AttributeName = InternalSpecific::TypeTrackerContent;
deprecated class AttributeName = InternalSpecific::TypeTrackerContent;
/** An attribute name, or the empty string (representing no attribute). */
class OptionalAttributeName = InternalSpecific::OptionalTypeTrackerContent;
deprecated class OptionalAttributeName = InternalSpecific::OptionalTypeTrackerContent;
/**
* DEPRECATED: Use `semmle.python.dataflow.new.TypeTracking` instead.
*
* The summary of the steps needed to track a value to a given dataflow node.
*
* This can be used to track objects that implement a certain API in order to
@@ -40,7 +44,7 @@ class OptionalAttributeName = InternalSpecific::OptionalTypeTrackerContent;
* `t = t2.step(myType(t2), result)`. If you additionally want to track individual
* intra-procedural steps, use `t = t2.smallstep(myCallback(t2), result)`.
*/
class TypeTracker extends Internal::TypeTracker {
deprecated class TypeTracker extends Internal::TypeTracker {
/**
* Holds if this is the starting point of type tracking, and the value starts in the attribute named `attrName`.
* The type tracking only ends after the attribute has been loaded.
@@ -55,12 +59,12 @@ class TypeTracker extends Internal::TypeTracker {
string getAttr() { result = this.getContent() }
}
module TypeTracker = Internal::TypeTracker;
deprecated module TypeTracker = Internal::TypeTracker;
class StepSummary = Internal::StepSummary;
deprecated class StepSummary = Internal::StepSummary;
module StepSummary = Internal::StepSummary;
deprecated module StepSummary = Internal::StepSummary;
class TypeBackTracker = Internal::TypeBackTracker;
deprecated class TypeBackTracker = Internal::TypeBackTracker;
module TypeBackTracker = Internal::TypeBackTracker;
deprecated module TypeBackTracker = Internal::TypeBackTracker;

View File

@@ -0,0 +1,56 @@
/**
* Provides classes and predicates for simple data-flow reachability suitable
* for tracking types.
*/
private import internal.TypeTrackingImpl as Impl
import Impl::Shared::TypeTracking<Impl::TypeTrackingInput>
/** A string that may appear as the name of an attribute or access path. */
class AttributeName = Impl::TypeTrackingInput::Content;
/**
* A summary of the steps needed to track a value to a given dataflow node.
*
* This can be used to track objects that implement a certain API in order to
* recognize calls to that API. Note that type-tracking does not by itself provide a
* source/sink relation, that is, it may determine that a node has a given type,
* but it won't determine where that type came from.
*
* It is recommended that all uses of this type are written in the following form,
* for tracking some type `myType`:
* ```ql
* Node myType(TypeTracker tt) {
* tt.start() and
* result = < source of myType >
* or
* exists(TypeTracker tt2 |
* tt = tt2.step(myType(tt2), result)
* )
* }
*
* Node myType() { myType(TypeTracker::end()).flowsTo(result) }
* ```
*
* If you want to track individual intra-procedural steps, use `tt2.smallstep`
* instead of `tt2.step`.
*/
class TypeTracker extends Impl::TypeTracker {
/**
* Holds if this is the starting point of type tracking, and the value starts in the attribute named `attrName`.
* The type tracking only ends after the attribute has been loaded.
*/
predicate startInAttr(string attrName) { this.startInContent(attrName) }
/**
* INTERNAL. DO NOT USE.
*
* Gets the attribute associated with this type tracker.
*/
string getAttr() {
result = this.getContent().asSome()
or
this.getContent().isNone() and
result = ""
}
}

View File

@@ -37,7 +37,7 @@ private import DataFlowPublic
private import DataFlowPrivate
private import FlowSummaryImpl as FlowSummaryImpl
private import semmle.python.internal.CachedStages
private import semmle.python.dataflow.new.internal.TypeTracker::CallGraphConstruction as CallGraphConstruction
private import semmle.python.dataflow.new.internal.TypeTrackingImpl::CallGraphConstruction as CallGraphConstruction
newtype TParameterPosition =
/** Used for `self` in methods, and `cls` in classmethods. */

View File

@@ -4,7 +4,7 @@
private import python
private import DataFlowPrivate
import semmle.python.dataflow.new.TypeTracker
import semmle.python.dataflow.new.TypeTracking
import Attributes
import LocalSources
private import semmle.python.essa.SsaCompute

View File

@@ -7,7 +7,7 @@
private import python
private import semmle.python.dataflow.new.DataFlow
private import semmle.python.dataflow.new.internal.ImportStar
private import semmle.python.dataflow.new.TypeTracker
private import semmle.python.dataflow.new.TypeTracking
private import semmle.python.dataflow.new.internal.DataFlowPrivate
/**

View File

@@ -72,6 +72,8 @@ class LocalSourceNode extends Node {
// We include all scope entry definitions, as these act as the local source within the scope they
// enter.
this.asCfgNode() = any(ScopeEntryDefinition def).getDefiningNode()
or
this instanceof ParameterNode
}
/** Holds if this `LocalSourceNode` can flow to `nodeTo` in one or more local flow steps. */
@@ -151,7 +153,7 @@ class LocalSourceNode extends Node {
* See `TypeBackTracker` for more details about how to use this.
*/
pragma[inline]
LocalSourceNode backtrack(TypeBackTracker t2, TypeBackTracker t) { t2 = t.step(result, this) }
LocalSourceNode backtrack(TypeBackTracker t2, TypeBackTracker t) { t = t2.step(result, this) }
}
/**
@@ -238,7 +240,7 @@ private module Cached {
* Helper predicate for `hasLocalSource`. Removes any steps go to module variable reads, as these
* are already local source nodes in their own right.
*/
cached
pragma[nomagic]
private predicate localSourceFlowStep(Node nodeFrom, Node nodeTo) {
simpleLocalFlowStep(nodeFrom, nodeTo) and
not nodeTo = any(ModuleVariableNode v).getARead()

View File

@@ -8,22 +8,22 @@ private module Cached {
* A description of a step on an inter-procedural data flow path.
*/
cached
newtype TStepSummary =
deprecated newtype TStepSummary =
LevelStep() or
CallStep() or
ReturnStep() or
StoreStep(TypeTrackerContent content) { basicStoreStep(_, _, content) } or
LoadStep(TypeTrackerContent content) { basicLoadStep(_, _, content) } or
LoadStoreStep(TypeTrackerContent load, TypeTrackerContent store) {
deprecated StoreStep(TypeTrackerContent content) { basicStoreStep(_, _, content) } or
deprecated LoadStep(TypeTrackerContent content) { basicLoadStep(_, _, content) } or
deprecated LoadStoreStep(TypeTrackerContent load, TypeTrackerContent store) {
basicLoadStoreStep(_, _, load, store)
} or
WithContent(ContentFilter filter) { basicWithContentStep(_, _, filter) } or
WithoutContent(ContentFilter filter) { basicWithoutContentStep(_, _, filter) } or
deprecated WithContent(ContentFilter filter) { basicWithContentStep(_, _, filter) } or
deprecated WithoutContent(ContentFilter filter) { basicWithoutContentStep(_, _, filter) } or
JumpStep()
cached
newtype TTypeTracker =
MkTypeTracker(Boolean hasCall, OptionalTypeTrackerContent content) {
deprecated newtype TTypeTracker =
deprecated MkTypeTracker(Boolean hasCall, OptionalTypeTrackerContent content) {
content = noContent()
or
// Restrict `content` to those that might eventually match a load.
@@ -40,8 +40,8 @@ private module Cached {
}
cached
newtype TTypeBackTracker =
MkTypeBackTracker(Boolean hasReturn, OptionalTypeTrackerContent content) {
deprecated newtype TTypeBackTracker =
deprecated MkTypeBackTracker(Boolean hasReturn, OptionalTypeTrackerContent content) {
content = noContent()
or
// As in MkTypeTracker, restrict `content` to those that might eventually match a store.
@@ -57,11 +57,13 @@ private module Cached {
/** Gets a type tracker with no content and the call bit set to the given value. */
cached
TypeTracker noContentTypeTracker(boolean hasCall) { result = MkTypeTracker(hasCall, noContent()) }
deprecated TypeTracker noContentTypeTracker(boolean hasCall) {
result = MkTypeTracker(hasCall, noContent())
}
/** Gets the summary resulting from appending `step` to type-tracking summary `tt`. */
cached
TypeTracker append(TypeTracker tt, StepSummary step) {
deprecated TypeTracker append(TypeTracker tt, StepSummary step) {
exists(Boolean hasCall, OptionalTypeTrackerContent currentContents |
tt = MkTypeTracker(hasCall, currentContents)
|
@@ -108,13 +110,13 @@ private module Cached {
}
pragma[nomagic]
private TypeBackTracker noContentTypeBackTracker(boolean hasReturn) {
deprecated private TypeBackTracker noContentTypeBackTracker(boolean hasReturn) {
result = MkTypeBackTracker(hasReturn, noContent())
}
/** Gets the summary resulting from prepending `step` to this type-tracking summary. */
cached
TypeBackTracker prepend(TypeBackTracker tbt, StepSummary step) {
deprecated TypeBackTracker prepend(TypeBackTracker tbt, StepSummary step) {
exists(Boolean hasReturn, OptionalTypeTrackerContent content |
tbt = MkTypeBackTracker(hasReturn, content)
|
@@ -167,7 +169,9 @@ private module Cached {
* Steps contained in this predicate should _not_ depend on the call graph.
*/
cached
predicate stepNoCall(TypeTrackingNode nodeFrom, TypeTrackingNode nodeTo, StepSummary summary) {
deprecated predicate stepNoCall(
TypeTrackingNode nodeFrom, TypeTrackingNode nodeTo, StepSummary summary
) {
exists(Node mid | nodeFrom.flowsTo(mid) and smallstepNoCall(mid, nodeTo, summary))
}
@@ -176,12 +180,14 @@ private module Cached {
* inter-procedural step from `nodeFrom` to `nodeTo`.
*/
cached
predicate stepCall(TypeTrackingNode nodeFrom, TypeTrackingNode nodeTo, StepSummary summary) {
deprecated predicate stepCall(
TypeTrackingNode nodeFrom, TypeTrackingNode nodeTo, StepSummary summary
) {
exists(Node mid | nodeFrom.flowsTo(mid) and smallstepCall(mid, nodeTo, summary))
}
cached
predicate smallstepNoCall(Node nodeFrom, TypeTrackingNode nodeTo, StepSummary summary) {
deprecated predicate smallstepNoCall(Node nodeFrom, TypeTrackingNode nodeTo, StepSummary summary) {
jumpStep(nodeFrom, nodeTo) and
summary = JumpStep()
or
@@ -210,7 +216,7 @@ private module Cached {
}
cached
predicate smallstepCall(Node nodeFrom, TypeTrackingNode nodeTo, StepSummary summary) {
deprecated predicate smallstepCall(Node nodeFrom, TypeTrackingNode nodeTo, StepSummary summary) {
callStep(nodeFrom, nodeTo) and summary = CallStep()
or
returnStep(nodeFrom, nodeTo) and
@@ -223,25 +229,27 @@ private module Cached {
private import Cached
private predicate step(TypeTrackingNode nodeFrom, TypeTrackingNode nodeTo, StepSummary summary) {
deprecated private predicate step(
TypeTrackingNode nodeFrom, TypeTrackingNode nodeTo, StepSummary summary
) {
stepNoCall(nodeFrom, nodeTo, summary)
or
stepCall(nodeFrom, nodeTo, summary)
}
pragma[nomagic]
private predicate stepProj(TypeTrackingNode nodeFrom, StepSummary summary) {
deprecated private predicate stepProj(TypeTrackingNode nodeFrom, StepSummary summary) {
step(nodeFrom, _, summary)
}
private predicate smallstep(Node nodeFrom, TypeTrackingNode nodeTo, StepSummary summary) {
deprecated private predicate smallstep(Node nodeFrom, TypeTrackingNode nodeTo, StepSummary summary) {
smallstepNoCall(nodeFrom, nodeTo, summary)
or
smallstepCall(nodeFrom, nodeTo, summary)
}
pragma[nomagic]
private predicate smallstepProj(Node nodeFrom, StepSummary summary) {
deprecated private predicate smallstepProj(Node nodeFrom, StepSummary summary) {
smallstep(nodeFrom, _, summary)
}
@@ -270,7 +278,7 @@ private predicate smallstepProj(Node nodeFrom, StepSummary summary) {
* function. This means we will track the fact that `x.attr` can have the type of `y` into the
* assignment to `z` inside `bar`, even though this attribute write happens _after_ `bar` is called.
*/
private predicate flowsToStoreStep(
deprecated private predicate flowsToStoreStep(
Node nodeFrom, TypeTrackingNode nodeTo, TypeTrackerContent content
) {
exists(Node obj | nodeTo.flowsTo(obj) and basicStoreStep(nodeFrom, obj, content))
@@ -279,7 +287,7 @@ private predicate flowsToStoreStep(
/**
* Holds if `loadContent` is loaded from `nodeFrom` and written to `storeContent` of `nodeTo`.
*/
private predicate flowsToLoadStoreStep(
deprecated private predicate flowsToLoadStoreStep(
Node nodeFrom, TypeTrackingNode nodeTo, TypeTrackerContent loadContent,
TypeTrackerContent storeContent
) {
@@ -293,7 +301,7 @@ private predicate flowsToLoadStoreStep(
*
* A description of a step on an inter-procedural data flow path.
*/
class StepSummary extends TStepSummary {
deprecated class StepSummary extends TStepSummary {
/** Gets a textual representation of this step summary. */
string toString() {
this instanceof LevelStep and result = "level"
@@ -316,7 +324,7 @@ class StepSummary extends TStepSummary {
}
/** Provides predicates for updating step summaries (`StepSummary`s). */
module StepSummary {
deprecated module StepSummary {
predicate append = Cached::append/2;
/**
@@ -411,6 +419,8 @@ module StepSummary {
}
/**
* DEPRECATED: Use `semmle.python.dataflow.new.TypeTracking` instead.
*
* A summary of the steps needed to track a value to a given dataflow node.
*
* This can be used to track objects that implement a certain API in order to
@@ -437,7 +447,7 @@ module StepSummary {
* `t = t2.step(myType(t2), result)`. If you additionally want to track individual
* intra-procedural steps, use `t = t2.smallstep(myCallback(t2), result)`.
*/
class TypeTracker extends TTypeTracker {
deprecated class TypeTracker extends TTypeTracker {
Boolean hasCall;
OptionalTypeTrackerContent content;
@@ -565,7 +575,7 @@ class TypeTracker extends TTypeTracker {
}
/** Provides predicates for implementing custom `TypeTracker`s. */
module TypeTracker {
deprecated module TypeTracker {
/**
* Gets a valid end point of type tracking.
*/
@@ -580,15 +590,17 @@ module TypeTracker {
}
pragma[nomagic]
private predicate backStepProj(TypeTrackingNode nodeTo, StepSummary summary) {
deprecated private predicate backStepProj(TypeTrackingNode nodeTo, StepSummary summary) {
step(_, nodeTo, summary)
}
private predicate backSmallstepProj(TypeTrackingNode nodeTo, StepSummary summary) {
deprecated private predicate backSmallstepProj(TypeTrackingNode nodeTo, StepSummary summary) {
smallstep(_, nodeTo, summary)
}
/**
* DEPRECATED: Use `semmle.python.dataflow.new.TypeTracking` instead.
*
* A summary of the steps needed to back-track a use of a value to a given dataflow node.
*
* This can for example be used to track callbacks that are passed to a certain API,
@@ -618,7 +630,7 @@ private predicate backSmallstepProj(TypeTrackingNode nodeTo, StepSummary summary
* `t2 = t.step(result, myCallback(t2))`. If you additionally want to track individual
* intra-procedural steps, use `t2 = t.smallstep(result, myCallback(t2))`.
*/
class TypeBackTracker extends TTypeBackTracker {
deprecated class TypeBackTracker extends TTypeBackTracker {
Boolean hasReturn;
OptionalTypeTrackerContent content;
@@ -747,7 +759,7 @@ class TypeBackTracker extends TTypeBackTracker {
}
/** Provides predicates for implementing custom `TypeBackTracker`s. */
module TypeBackTracker {
deprecated module TypeBackTracker {
/**
* Gets a valid end point of type back-tracking.
*/
@@ -768,14 +780,14 @@ module TypeBackTracker {
* `stepCall` relation (`stepNoCall` not being recursive, can be join-ordered in the
* same way as in `stepInlineLate`).
*/
module CallGraphConstruction {
deprecated module CallGraphConstruction {
/** The input to call graph construction. */
signature module InputSig {
/** A state to track during type tracking. */
class State;
/** Holds if type tracking should start at `start` in state `state`. */
predicate start(Node start, State state);
deprecated predicate start(Node start, State state);
/**
* Holds if type tracking should use the step from `nodeFrom` to `nodeTo`,
@@ -784,7 +796,7 @@ module CallGraphConstruction {
* Implementing this predicate using `StepSummary::[small]stepNoCall` yields
* standard type tracking.
*/
predicate stepNoCall(Node nodeFrom, Node nodeTo, StepSummary summary);
deprecated predicate stepNoCall(Node nodeFrom, Node nodeTo, StepSummary summary);
/**
* Holds if type tracking should use the step from `nodeFrom` to `nodeTo`,
@@ -793,7 +805,7 @@ module CallGraphConstruction {
* Implementing this predicate using `StepSummary::[small]stepCall` yields
* standard type tracking.
*/
predicate stepCall(Node nodeFrom, Node nodeTo, StepSummary summary);
deprecated predicate stepCall(Node nodeFrom, Node nodeTo, StepSummary summary);
/** A projection of an element from the state space. */
class StateProj;
@@ -802,25 +814,25 @@ module CallGraphConstruction {
StateProj stateProj(State state);
/** Holds if type tracking should stop at `n` when we are tracking projected state `stateProj`. */
predicate filter(Node n, StateProj stateProj);
deprecated predicate filter(Node n, StateProj stateProj);
}
/** Provides the `track` predicate for use in call graph construction. */
module Make<InputSig Input> {
pragma[nomagic]
private predicate stepNoCallProj(Node nodeFrom, StepSummary summary) {
deprecated private predicate stepNoCallProj(Node nodeFrom, StepSummary summary) {
Input::stepNoCall(nodeFrom, _, summary)
}
pragma[nomagic]
private predicate stepCallProj(Node nodeFrom, StepSummary summary) {
deprecated private predicate stepCallProj(Node nodeFrom, StepSummary summary) {
Input::stepCall(nodeFrom, _, summary)
}
bindingset[nodeFrom, t]
pragma[inline_late]
pragma[noopt]
private TypeTracker stepNoCallInlineLate(
deprecated private TypeTracker stepNoCallInlineLate(
TypeTracker t, TypeTrackingNode nodeFrom, TypeTrackingNode nodeTo
) {
exists(StepSummary summary |
@@ -837,7 +849,7 @@ module CallGraphConstruction {
}
pragma[nomagic]
private Node track(Input::State state, TypeTracker t) {
deprecated private Node track(Input::State state, TypeTracker t) {
t.start() and Input::start(result, state)
or
exists(Input::StateProj stateProj |
@@ -855,12 +867,12 @@ module CallGraphConstruction {
bindingset[t, summary]
pragma[inline_late]
private TypeTracker appendInlineLate(TypeTracker t, StepSummary summary) {
deprecated private TypeTracker appendInlineLate(TypeTracker t, StepSummary summary) {
result = t.append(summary)
}
pragma[nomagic]
private Node trackCall(Input::State state, TypeTracker t, StepSummary summary) {
deprecated private Node trackCall(Input::State state, TypeTracker t, StepSummary summary) {
exists(TypeTracker t2 |
// non-linear recursion
result = track(state, t2) and
@@ -871,7 +883,7 @@ module CallGraphConstruction {
/** Gets a node that can be reached from _some_ start node in state `state`. */
pragma[nomagic]
Node track(Input::State state) { result = track(state, TypeTracker::end()) }
deprecated Node track(Input::State state) { result = track(state, TypeTracker::end()) }
}
/** A simple version of `CallGraphConstruction` that uses standard type tracking. */
@@ -882,15 +894,15 @@ module CallGraphConstruction {
class State;
/** Holds if type tracking should start at `start` in state `state`. */
predicate start(Node start, State state);
deprecated predicate start(Node start, State state);
/** Holds if type tracking should stop at `n`. */
predicate filter(Node n);
deprecated predicate filter(Node n);
}
/** Provides the `track` predicate for use in call graph construction. */
module Make<InputSig Input> {
private module I implements CallGraphConstruction::InputSig {
deprecated private module I implements CallGraphConstruction::InputSig {
private import codeql.util.Unit
class State = Input::State;
@@ -915,7 +927,7 @@ module CallGraphConstruction {
}
}
import CallGraphConstruction::Make<I>
deprecated import CallGraphConstruction::Make<I>
}
}
}

View File

@@ -5,86 +5,54 @@
private import python
private import semmle.python.dataflow.new.internal.DataFlowPublic as DataFlowPublic
private import semmle.python.dataflow.new.internal.DataFlowPrivate as DataFlowPrivate
private import TypeTrackingImpl as TypeTrackingImpl
import semmle.python.internal.CachedStages
class Node = DataFlowPublic::Node;
deprecated class Node = DataFlowPublic::Node;
class TypeTrackingNode = DataFlowPublic::TypeTrackingNode;
deprecated class TypeTrackingNode = DataFlowPublic::TypeTrackingNode;
/** A content name for use by type trackers, or the empty string. */
class OptionalTypeTrackerContent extends string {
deprecated class OptionalTypeTrackerContent extends string {
OptionalTypeTrackerContent() {
this = ""
or
this = getPossibleContentName()
this instanceof TypeTrackingImpl::TypeTrackingInput::Content
}
}
/** A content name for use by type trackers. */
class TypeTrackerContent extends OptionalTypeTrackerContent {
deprecated class TypeTrackerContent extends OptionalTypeTrackerContent {
TypeTrackerContent() { this != "" }
}
/** Gets the content string representing no value. */
OptionalTypeTrackerContent noContent() { result = "" }
deprecated OptionalTypeTrackerContent noContent() { result = "" }
/**
* A label to use for `WithContent` and `WithoutContent` steps, restricting
* which `ContentSet` may pass through. Not currently used in Python.
*/
class ContentFilter extends Unit {
deprecated class ContentFilter extends Unit {
TypeTrackerContent getAMatchingContent() { none() }
}
pragma[inline]
predicate compatibleContents(TypeTrackerContent storeContent, TypeTrackerContent loadContent) {
deprecated predicate compatibleContents(
TypeTrackerContent storeContent, TypeTrackerContent loadContent
) {
storeContent = loadContent
}
predicate simpleLocalFlowStep = DataFlowPrivate::simpleLocalFlowStepForTypetracking/2;
predicate simpleLocalFlowStep = TypeTrackingImpl::TypeTrackingInput::simpleLocalSmallStep/2;
predicate jumpStep(Node nodeFrom, Node nodeTo) {
DataFlowPrivate::jumpStepSharedWithTypeTracker(nodeFrom, nodeTo)
or
capturedJumpStep(nodeFrom, nodeTo)
}
predicate capturedJumpStep(Node nodeFrom, Node nodeTo) {
// Jump into a capturing scope.
//
// var = expr
// ...
// def f():
// ..var is used..
//
// nodeFrom is `expr`
// nodeTo is entry node for `f`
exists(ScopeEntryDefinition e, SsaSourceVariable var, DefinitionNode def |
e.getSourceVariable() = var and
var.hasDefiningNode(def)
|
nodeTo.asCfgNode() = e.getDefiningNode() and
nodeFrom.asCfgNode() = def.getValue() and
var.getScope().getScope*() = nodeFrom.getScope()
)
}
predicate jumpStep = TypeTrackingImpl::TypeTrackingInput::jumpStep/2;
/** Holds if there is a level step from `nodeFrom` to `nodeTo`, which may depend on the call graph. */
predicate levelStepCall(Node nodeFrom, Node nodeTo) { none() }
deprecated predicate levelStepCall(Node nodeFrom, Node nodeTo) { none() }
/** Holds if there is a level step from `nodeFrom` to `nodeTo`, which does not depend on the call graph. */
predicate levelStepNoCall(Node nodeFrom, Node nodeTo) {
TypeTrackerSummaryFlow::levelStepNoCall(nodeFrom, nodeTo)
}
/**
* Gets the name of a possible piece of content. For Python, this is currently only attribute names,
* using the name of the attribute for the corresponding content.
*/
string getPossibleContentName() {
Stages::TypeTracking::ref() and // the TypeTracking::append() etc. predicates that we want to cache depend on this predicate, so we can place the `ref()` call here to get around identical files.
result = any(DataFlowPublic::AttrRef a).getAttributeName()
}
predicate levelStepNoCall = TypeTrackingImpl::TypeTrackingInput::levelStepNoCall/2;
/**
* Holds if `nodeFrom` steps to `nodeTo` by being passed as a parameter in a call.
@@ -93,177 +61,43 @@ string getPossibleContentName() {
* recursion (or, at best, terrible performance), since identifying calls to library
* methods is done using API graphs (which uses type tracking).
*/
predicate callStep(DataFlowPublic::ArgumentNode nodeFrom, DataFlowPublic::ParameterNode nodeTo) {
exists(
DataFlowPrivate::DataFlowCall call, DataFlowPrivate::DataFlowCallable callable,
DataFlowPrivate::ArgumentPosition apos, DataFlowPrivate::ParameterPosition ppos
|
nodeFrom = call.getArgument(apos) and
nodeTo = callable.getParameter(ppos) and
DataFlowPrivate::parameterMatch(ppos, apos) and
callable = call.getCallable()
)
}
predicate callStep = TypeTrackingImpl::TypeTrackingInput::callStep/2;
/** Holds if `nodeFrom` steps to `nodeTo` by being returned from a call. */
predicate returnStep(DataFlowPrivate::ReturnNode nodeFrom, Node nodeTo) {
exists(DataFlowPrivate::ExtractedDataFlowCall call |
nodeFrom.getEnclosingCallable() = call.getCallable() and
nodeTo.(DataFlowPublic::CfgNode).getNode() = call.getNode()
)
}
predicate returnStep = TypeTrackingImpl::TypeTrackingInput::returnStep/2;
/**
* Holds if `nodeFrom` is being written to the `content` content of the object in `nodeTo`.
*/
predicate basicStoreStep(Node nodeFrom, Node nodeTo, string content) {
exists(DataFlowPublic::AttrWrite a |
a.mayHaveAttributeName(content) and
nodeFrom = a.getValue() and
nodeTo = a.getObject()
)
or
exists(DataFlowPublic::ContentSet contents |
contents.(DataFlowPublic::AttributeContent).getAttribute() = content
|
TypeTrackerSummaryFlow::basicStoreStep(nodeFrom, nodeTo, contents)
)
}
predicate basicStoreStep = TypeTrackingImpl::TypeTrackingInput::storeStep/3;
/**
* Holds if `nodeTo` is the result of accessing the `content` content of `nodeFrom`.
*/
predicate basicLoadStep(Node nodeFrom, Node nodeTo, string content) {
exists(DataFlowPublic::AttrRead a |
a.mayHaveAttributeName(content) and
nodeFrom = a.getObject() and
nodeTo = a
)
or
exists(DataFlowPublic::ContentSet contents |
contents.(DataFlowPublic::AttributeContent).getAttribute() = content
|
TypeTrackerSummaryFlow::basicLoadStep(nodeFrom, nodeTo, contents)
)
}
predicate basicLoadStep = TypeTrackingImpl::TypeTrackingInput::loadStep/3;
/**
* Holds if the `loadContent` of `nodeFrom` is stored in the `storeContent` of `nodeTo`.
*/
predicate basicLoadStoreStep(Node nodeFrom, Node nodeTo, string loadContent, string storeContent) {
exists(DataFlowPublic::ContentSet loadContents, DataFlowPublic::ContentSet storeContents |
loadContents.(DataFlowPublic::AttributeContent).getAttribute() = loadContent and
storeContents.(DataFlowPublic::AttributeContent).getAttribute() = storeContent
|
TypeTrackerSummaryFlow::basicLoadStoreStep(nodeFrom, nodeTo, loadContents, storeContents)
)
}
predicate basicLoadStoreStep = TypeTrackingImpl::TypeTrackingInput::loadStoreStep/4;
/**
* Holds if type-tracking should step from `nodeFrom` to `nodeTo` but block flow of contents matched by `filter` through here.
*/
predicate basicWithoutContentStep(Node nodeFrom, Node nodeTo, ContentFilter filter) { none() }
deprecated predicate basicWithoutContentStep(Node nodeFrom, Node nodeTo, ContentFilter filter) {
none()
}
/**
* Holds if type-tracking should step from `nodeFrom` to `nodeTo` if inside a content matched by `filter`.
*/
predicate basicWithContentStep(Node nodeFrom, Node nodeTo, ContentFilter filter) { none() }
deprecated predicate basicWithContentStep(Node nodeFrom, Node nodeTo, ContentFilter filter) {
none()
}
/**
* A utility class that is equivalent to `boolean` but does not require type joining.
*/
class Boolean extends boolean {
deprecated class Boolean extends boolean {
Boolean() { this = true or this = false }
}
private import SummaryTypeTracker as SummaryTypeTracker
private import semmle.python.dataflow.new.internal.FlowSummaryImpl as FlowSummaryImpl
private import semmle.python.dataflow.new.internal.DataFlowDispatch as DataFlowDispatch
pragma[noinline]
private predicate argumentPositionMatch(
DataFlowPublic::CallCfgNode call, DataFlowPublic::Node arg,
DataFlowDispatch::ParameterPosition ppos
) {
exists(DataFlowDispatch::ArgumentPosition apos |
DataFlowDispatch::parameterMatch(ppos, apos) and
DataFlowDispatch::normalCallArg(call.getNode(), arg, apos)
)
}
private module SummaryTypeTrackerInput implements SummaryTypeTracker::Input {
// Dataflow nodes
class Node = DataFlowPublic::Node;
// Content
class TypeTrackerContent = DataFlowPublic::ContentSet;
class TypeTrackerContentFilter = ContentFilter;
TypeTrackerContentFilter getFilterFromWithoutContentStep(TypeTrackerContent content) { none() }
TypeTrackerContentFilter getFilterFromWithContentStep(TypeTrackerContent content) { none() }
// Callables
class SummarizedCallable = FlowSummaryImpl::Private::SummarizedCallableImpl;
// Summaries and their stacks
class SummaryComponent = FlowSummaryImpl::Private::SummaryComponent;
class SummaryComponentStack = FlowSummaryImpl::Private::SummaryComponentStack;
predicate singleton = FlowSummaryImpl::Private::SummaryComponentStack::singleton/1;
predicate push = FlowSummaryImpl::Private::SummaryComponentStack::push/2;
// Relating content to summaries
predicate content = FlowSummaryImpl::Private::SummaryComponent::content/1;
SummaryComponent withoutContent(TypeTrackerContent contents) { none() }
SummaryComponent withContent(TypeTrackerContent contents) { none() }
predicate return = FlowSummaryImpl::Private::SummaryComponent::return/0;
// Relating nodes to summaries
Node argumentOf(Node call, SummaryComponent arg, boolean isPostUpdate) {
exists(DataFlowDispatch::ParameterPosition pos |
arg = FlowSummaryImpl::Private::SummaryComponent::argument(pos) and
argumentPositionMatch(call, result, pos) and
isPostUpdate = [false, true] // todo: implement when/if Python uses post-update nodes in type tracking
)
}
Node parameterOf(Node callable, SummaryComponent param) {
exists(
DataFlowDispatch::ArgumentPosition apos, DataFlowDispatch::ParameterPosition ppos, Parameter p
|
param = FlowSummaryImpl::Private::SummaryComponent::parameter(apos) and
DataFlowDispatch::parameterMatch(ppos, apos) and
result.asCfgNode().getNode() = p and
(
exists(int i | ppos.isPositional(i) |
p = callable.getALocalSource().asExpr().(CallableExpr).getInnerScope().getArg(i)
)
or
exists(string name | ppos.isKeyword(name) |
p = callable.getALocalSource().asExpr().(CallableExpr).getInnerScope().getArgByName(name)
)
)
)
}
Node returnOf(Node callable, SummaryComponent return) {
return = FlowSummaryImpl::Private::SummaryComponent::return() and
// `result` should be the return value of a callable expression (lambda or function) referenced by `callable`
result.asCfgNode() =
callable.getALocalSource().asExpr().(CallableExpr).getInnerScope().getAReturnValueFlowNode()
}
// Relating callables to nodes
Node callTo(SummarizedCallable callable) {
result = callable.(DataFlowDispatch::LibraryCallable).getACallSimple()
}
}
private module TypeTrackerSummaryFlow = SummaryTypeTracker::SummaryFlow<SummaryTypeTrackerInput>;

View File

@@ -0,0 +1,274 @@
import codeql.util.Unit
import codeql.typetracking.TypeTracking as Shared
import codeql.typetracking.internal.TypeTrackingImpl as SharedImpl
private import python
private import semmle.python.internal.CachedStages
private import semmle.python.dataflow.new.internal.DataFlowPublic as DataFlowPublic
private import semmle.python.dataflow.new.internal.DataFlowPrivate as DataFlowPrivate
private import SummaryTypeTracker as SummaryTypeTracker
private import semmle.python.dataflow.new.internal.FlowSummaryImpl as FlowSummaryImpl
private import semmle.python.dataflow.new.internal.DataFlowDispatch as DataFlowDispatch
private module SummaryTypeTrackerInput implements SummaryTypeTracker::Input {
// Dataflow nodes
class Node = DataFlowPublic::Node;
// Content
class TypeTrackerContent = DataFlowPublic::ContentSet;
class TypeTrackerContentFilter = TypeTrackingInput::ContentFilter;
TypeTrackerContentFilter getFilterFromWithoutContentStep(TypeTrackerContent content) { none() }
TypeTrackerContentFilter getFilterFromWithContentStep(TypeTrackerContent content) { none() }
// Callables
class SummarizedCallable = FlowSummaryImpl::Private::SummarizedCallableImpl;
// Summaries and their stacks
class SummaryComponent = FlowSummaryImpl::Private::SummaryComponent;
class SummaryComponentStack = FlowSummaryImpl::Private::SummaryComponentStack;
predicate singleton = FlowSummaryImpl::Private::SummaryComponentStack::singleton/1;
predicate push = FlowSummaryImpl::Private::SummaryComponentStack::push/2;
// Relating content to summaries
predicate content = FlowSummaryImpl::Private::SummaryComponent::content/1;
SummaryComponent withoutContent(TypeTrackerContent contents) { none() }
SummaryComponent withContent(TypeTrackerContent contents) { none() }
predicate return = FlowSummaryImpl::Private::SummaryComponent::return/0;
pragma[noinline]
private predicate argumentPositionMatch(
DataFlowPublic::CallCfgNode call, DataFlowPublic::Node arg,
DataFlowDispatch::ParameterPosition ppos
) {
exists(DataFlowDispatch::ArgumentPosition apos |
DataFlowDispatch::parameterMatch(ppos, apos) and
DataFlowDispatch::normalCallArg(call.getNode(), arg, apos)
)
}
// Relating nodes to summaries
Node argumentOf(Node call, SummaryComponent arg, boolean isPostUpdate) {
exists(DataFlowDispatch::ParameterPosition pos |
arg = FlowSummaryImpl::Private::SummaryComponent::argument(pos) and
argumentPositionMatch(call, result, pos) and
isPostUpdate = [false, true] // todo: implement when/if Python uses post-update nodes in type tracking
)
}
Node parameterOf(Node callable, SummaryComponent param) {
exists(
DataFlowDispatch::ArgumentPosition apos, DataFlowDispatch::ParameterPosition ppos, Parameter p
|
param = FlowSummaryImpl::Private::SummaryComponent::parameter(apos) and
DataFlowDispatch::parameterMatch(ppos, apos) and
result.asCfgNode().getNode() = p and
(
exists(int i | ppos.isPositional(i) |
p = callable.getALocalSource().asExpr().(CallableExpr).getInnerScope().getArg(i)
)
or
exists(string name | ppos.isKeyword(name) |
p = callable.getALocalSource().asExpr().(CallableExpr).getInnerScope().getArgByName(name)
)
)
)
}
Node returnOf(Node callable, SummaryComponent return) {
return = FlowSummaryImpl::Private::SummaryComponent::return() and
// `result` should be the return value of a callable expression (lambda or function) referenced by `callable`
result.asCfgNode() =
callable.getALocalSource().asExpr().(CallableExpr).getInnerScope().getAReturnValueFlowNode()
}
// Relating callables to nodes
Node callTo(SummarizedCallable callable) {
result = callable.(DataFlowDispatch::LibraryCallable).getACallSimple()
}
}
private module TypeTrackerSummaryFlow = SummaryTypeTracker::SummaryFlow<SummaryTypeTrackerInput>;
/**
* Gets the name of a possible piece of content. For Python, this is currently only attribute names,
* using the name of the attribute for the corresponding content.
*/
private string getPossibleContentName() {
Stages::TypeTracking::ref() and // the TypeTracking::append() etc. predicates that we want to cache depend on this predicate, so we can place the `ref()` call here to get around identical files.
result = any(DataFlowPublic::AttrRef a).getAttributeName()
}
module TypeTrackingInput implements Shared::TypeTrackingInput {
class Node = DataFlowPublic::Node;
class LocalSourceNode = DataFlowPublic::LocalSourceNode;
class Content instanceof string {
Content() { this = getPossibleContentName() }
string toString() { result = this }
}
/**
* A label to use for `WithContent` and `WithoutContent` steps, restricting
* which `ContentSet` may pass through.
*/
class ContentFilter extends Unit {
Content getAMatchingContent() { none() }
}
/**
* Holds if a value stored with `storeContents` can be read back with `loadContents`.
*/
pragma[inline]
predicate compatibleContents(Content storeContents, Content loadContents) {
storeContents = loadContents
}
/** Holds if there is a simple local flow step from `nodeFrom` to `nodeTo` */
predicate simpleLocalSmallStep = DataFlowPrivate::simpleLocalFlowStepForTypetracking/2;
/** Holds if there is a level step from `nodeFrom` to `nodeTo`, which may depend on the call graph. */
predicate levelStepCall(Node nodeFrom, LocalSourceNode nodeTo) { none() }
/** Holds if there is a level step from `nodeFrom` to `nodeTo`, which does not depend on the call graph. */
predicate levelStepNoCall(Node nodeFrom, LocalSourceNode nodeTo) {
TypeTrackerSummaryFlow::levelStepNoCall(nodeFrom, nodeTo)
}
/**
* Holds if `nodeFrom` steps to `nodeTo` by being passed as a parameter in a call.
*
* Flow into summarized library methods is not included, as that will lead to negative
* recursion (or, at best, terrible performance), since identifying calls to library
* methods is done using API graphs (which uses type tracking).
*/
predicate callStep(Node nodeFrom, LocalSourceNode nodeTo) {
exists(
DataFlowPrivate::DataFlowCall call, DataFlowPrivate::DataFlowCallable callable,
DataFlowPrivate::ArgumentPosition apos, DataFlowPrivate::ParameterPosition ppos
|
nodeFrom = call.getArgument(apos) and
nodeTo = callable.getParameter(ppos) and
DataFlowPrivate::parameterMatch(ppos, apos) and
callable = call.getCallable()
)
}
/**
* Holds if `nodeFrom` steps to `nodeTo` by being returned from a call.
*
* Flow out of summarized library methods is not included, as that will lead to negative
* recursion (or, at best, terrible performance), since identifying calls to library
* methods is done using API graphs (which uses type tracking).
*/
predicate returnStep(Node nodeFrom, LocalSourceNode nodeTo) {
exists(DataFlowPrivate::ExtractedDataFlowCall call |
nodeFrom.(DataFlowPrivate::ReturnNode).getEnclosingCallable() = call.getCallable() and
nodeTo.(DataFlowPublic::CfgNode).getNode() = call.getNode()
)
}
/**
* Holds if `nodeFrom` is being written to the `content` content of the object in `nodeTo`.
*/
predicate storeStep(Node nodeFrom, Node nodeTo, Content content) {
exists(DataFlowPublic::AttrWrite a |
a.mayHaveAttributeName(content) and
nodeFrom = a.getValue() and
nodeTo = a.getObject()
)
or
exists(DataFlowPublic::ContentSet contents |
contents.(DataFlowPublic::AttributeContent).getAttribute() = content
|
TypeTrackerSummaryFlow::basicStoreStep(nodeFrom, nodeTo, contents)
)
}
/**
* Holds if `nodeTo` is the result of accessing the `content` content of `nodeFrom`.
*/
predicate loadStep(Node nodeFrom, LocalSourceNode nodeTo, Content content) {
exists(DataFlowPublic::AttrRead a |
a.mayHaveAttributeName(content) and
nodeFrom = a.getObject() and
nodeTo = a
)
or
exists(DataFlowPublic::ContentSet contents |
contents.(DataFlowPublic::AttributeContent).getAttribute() = content
|
TypeTrackerSummaryFlow::basicLoadStep(nodeFrom, nodeTo, contents)
)
}
/**
* Holds if the `loadContent` of `nodeFrom` is stored in the `storeContent` of `nodeTo`.
*/
predicate loadStoreStep(Node nodeFrom, Node nodeTo, Content loadContent, Content storeContent) {
exists(DataFlowPublic::ContentSet loadContents, DataFlowPublic::ContentSet storeContents |
loadContents.(DataFlowPublic::AttributeContent).getAttribute() = loadContent and
storeContents.(DataFlowPublic::AttributeContent).getAttribute() = storeContent
|
TypeTrackerSummaryFlow::basicLoadStoreStep(nodeFrom, nodeTo, loadContents, storeContents)
)
}
/**
* Holds if type-tracking should step from `nodeFrom` to `nodeTo` if inside a content matched by `filter`.
*/
predicate withContentStep(Node nodeFrom, LocalSourceNode nodeTo, ContentFilter filter) {
TypeTrackerSummaryFlow::basicWithContentStep(nodeFrom, nodeTo, filter)
}
/**
* Holds if type-tracking should step from `nodeFrom` to `nodeTo` but block flow of contents matched by `filter` through here.
*/
predicate withoutContentStep(Node nodeFrom, LocalSourceNode nodeTo, ContentFilter filter) {
TypeTrackerSummaryFlow::basicWithoutContentStep(nodeFrom, nodeTo, filter)
}
private predicate capturedJumpStep(Node nodeFrom, Node nodeTo) {
// Jump into a capturing scope.
//
// var = expr
// ...
// def f():
// ..var is used..
//
// nodeFrom is `expr`
// nodeTo is entry node for `f`
exists(ScopeEntryDefinition e, SsaSourceVariable var, DefinitionNode def |
e.getSourceVariable() = var and
var.hasDefiningNode(def)
|
nodeTo.asCfgNode() = e.getDefiningNode() and
nodeFrom.asCfgNode() = def.getValue() and
var.getScope().getScope*() = nodeFrom.getScope()
)
}
/**
* Holds if data can flow from `node1` to `node2` in a way that discards call contexts.
*/
predicate jumpStep(Node nodeFrom, LocalSourceNode nodeTo) {
DataFlowPrivate::jumpStepSharedWithTypeTracker(nodeFrom, nodeTo)
or
capturedJumpStep(nodeFrom, nodeTo)
}
predicate hasFeatureBacktrackStoreTarget() { any() }
predicate nonStandardFlowsTo(LocalSourceNode localSource, Node dst) { localSource.flowsTo(dst) }
}
import SharedImpl::TypeTracking<TypeTrackingInput>

View File

@@ -111,6 +111,7 @@ module Stages {
predicate ref() { 1 = 1 }
private import semmle.python.dataflow.new.DataFlow::DataFlow as NewDataFlow
private import semmle.python.dataflow.new.internal.TypeTrackingImpl as TypeTrackingImpl
private import semmle.python.ApiGraphs::API as API
/**
@@ -121,7 +122,7 @@ module Stages {
predicate backref() {
1 = 1
or
exists(any(NewDataFlow::TypeTracker t).append(_))
exists(TypeTrackingImpl::append(_, _))
or
exists(any(API::Node n).getAMember().getAValueReachableFromSource())
}

View File

@@ -50,7 +50,7 @@ module UnsafeShellCommandConstruction {
source = backtrackShellExec(TypeTracker::TypeBackTracker::end(), shellExec)
}
import semmle.python.dataflow.new.TypeTracker as TypeTracker
import semmle.python.dataflow.new.TypeTracking as TypeTracker
private DataFlow::LocalSourceNode backtrackShellExec(
TypeTracker::TypeBackTracker t, Concepts::SystemCommandExecution shellExec

View File

@@ -8,7 +8,7 @@ private import semmle.python.ApiGraphs
* `getACall` predicate on `SummarizedCallable`.
*/
module RecursionGuard {
private import semmle.python.dataflow.new.internal.TypeTrackerSpecific as TT
private import semmle.python.dataflow.new.internal.TypeTrackingImpl::TypeTrackingInput as TT
private class RecursionGuard extends SummarizedCallable {
RecursionGuard() { this = "RecursionGuard" }

View File

@@ -8,7 +8,7 @@ private import semmle.python.ApiGraphs
* `getACall` predicate on `SummarizedCallable`.
*/
module RecursionGuard {
private import semmle.python.dataflow.new.internal.TypeTrackerSpecific as TT
private import semmle.python.dataflow.new.internal.TypeTrackingImpl::TypeTrackingInput as TT
private class RecursionGuard extends SummarizedCallable {
RecursionGuard() { this = "TypeTrackingSummariesRecursionGuard" }

View File

@@ -1,6 +1,6 @@
import python
import semmle.python.dataflow.new.DataFlow
import semmle.python.dataflow.new.TypeTracker
import semmle.python.dataflow.new.TypeTracking
import TestUtilities.InlineExpectationsTest
import semmle.python.ApiGraphs
import TestSummaries

View File

@@ -1,6 +1,6 @@
import python
import semmle.python.dataflow.new.DataFlow
import semmle.python.dataflow.new.TypeTracker
import semmle.python.dataflow.new.TypeTracking
import semmle.python.ApiGraphs
private DataFlow::TypeTrackingNode module_tracker(TypeTracker t) {

View File

@@ -1,6 +1,6 @@
import python
import semmle.python.dataflow.new.DataFlow
import semmle.python.dataflow.new.TypeTracker
import semmle.python.dataflow.new.TypeTracking
import TestUtilities.InlineExpectationsTest
import semmle.python.ApiGraphs