hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2026-04-30 19:26:02 +02:00
Code Issues Packages Projects Releases Wiki Activity

Python: Autoformat rest of semmle/python.

Browse Source
This commit is contained in:
Taus Brock-Nannestad
2020-03-20 16:42:22 +01:00
parent 810e91ea00
commit 9044ff6959
17 changed files with 758 additions and 1047 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
python/ql/src/semmle/python/dataflow/Implementation.qll
View File

@@ -57,8 +57,8 @@ private newtype TAttributePath =
* It might make sense to add another level, attribute of attribute.
* But some experimentation would be needed.
*/
TAttribute(string name) { exists(Attribute a | a.getName() = name) }
TAttribute(string name) { exists(Attribute a | a.getName() = name) }
/**
* The attribute of the tracked value holding the taint.
@@ -445,9 +445,8 @@ class TaintTrackingImplementation extends string {
context = TNoParam() and
src = TTaintTrackingNode_(retval, TNoParam(), path, kind, this) and
node.asCfgNode() = call and
retval.asCfgNode() = any(Return ret | ret.getScope() = pyfunc.getScope())
.getValue()
.getAFlowNode()
retval.asCfgNode() =
any(Return ret | ret.getScope() = pyfunc.getScope()).getValue().getAFlowNode()
) and
edgeLabel = "return"
}
@@ -469,9 +468,8 @@ class TaintTrackingImplementation extends string {
this.callContexts(call, src, pyfunc, context, callee) and
retnode = TTaintTrackingNode_(retval, callee, path, kind, this) and
node.asCfgNode() = call and
retval.asCfgNode() = any(Return ret | ret.getScope() = pyfunc.getScope())
.getValue()
.getAFlowNode()
retval.asCfgNode() =
any(Return ret | ret.getScope() = pyfunc.getScope()).getValue().getAFlowNode()
) and
edgeLabel = "call"
}
@@ -717,7 +715,8 @@ private class EssaTaintTracking extends string {
path.noAttribute()
|
assign.getValue().getAFlowNode() = srcnode.asCfgNode() and
kind = iterable_unpacking_descent(assign.getATarget().getAFlowNode(), defn.getDefiningNode(),
kind =
iterable_unpacking_descent(assign.getATarget().getAFlowNode(), defn.getDefiningNode(),
srckind)
)
}
@@ -734,8 +733,8 @@ private class EssaTaintTracking extends string {
then result = parent_kind
else result = parent_kind.getMember()
or
result = iterable_unpacking_descent(left_parent.getAnElement(), left_defn,
parent_kind.getMember())
result =
iterable_unpacking_descent(left_parent.getAnElement(), left_defn, parent_kind.getMember())
}
pragma[noinline]

78
python/ql/src/semmle/python/dependencies/Dependencies.qll
View File

@@ -1,13 +1,14 @@
import python
import semmle.python.dependencies.DependencyKind
private predicate importDependency(Object target, AstNode source) {
source.getScope() != target.getOrigin() and /* Imports of own module are ignored */
source.getScope() != target.getOrigin() and
/* Imports of own module are ignored */
(
exists(ModuleObject importee, ImportingStmt imp_stmt |
source = imp_stmt and
importee = target |
importee = target
|
exists(ImportMember im | imp_stmt.contains(im) |
importee.importedAs(im.getImportedModuleName())
)
@@ -23,11 +24,9 @@ private predicate importDependency(Object target, AstNode source) {
)
or
/* from m import name, where m.name is not a submodule */
exists(PythonModuleObject importee, ImportingStmt imp_stmt |
source = imp_stmt |
exists(PythonModuleObject importee, ImportingStmt imp_stmt | source = imp_stmt |
exists(ImportMember im | imp_stmt.contains(im) |
importee.importedAs(im.getModule().(ImportExpr).getImportedModuleName())
and
importee.importedAs(im.getModule().(ImportExpr).getImportedModuleName()) and
defn_of_module_attribute(target, importee.getModule(), im.getName())
)
)
@@ -35,16 +34,12 @@ private predicate importDependency(Object target, AstNode source) {
}
class PythonImport extends DependencyKind {
PythonImport() {
this = "import"
}
PythonImport() { this = "import" }
override predicate isADependency(AstNode source, Object target) {
this = this and
importDependency(target, source)
}
}
private predicate interesting(Object target) {
@@ -58,10 +53,7 @@ private predicate interesting(Object target) {
}
class PythonUse extends DependencyKind {
PythonUse() {
this = "use"
}
PythonUse() { this = "use" }
override predicate isADependency(AstNode source, Object target) {
interesting(target) and
@@ -71,22 +63,20 @@ class PythonUse extends DependencyKind {
use.getNode() = source and
use.refersTo(obj) and
use.isLoad()
|
|
interesting(obj) and target = obj
)
and
) and
not has_more_specific_dependency_source(source)
}
}
/** Whether there is a more specific dependency source than this one.
/**
* Whether there is a more specific dependency source than this one.
* E.g. if the expression pack.mod.func is a dependency on the function 'func' in 'pack.mod'
* don't make pack.mod depend on the module 'pack.mod'
*/
private predicate has_more_specific_dependency_source(Expr e) {
exists(Attribute member |
member.getObject() = e |
exists(Attribute member | member.getObject() = e |
attribute_access_dependency(_, member)
or
has_more_specific_dependency_source(member)
@@ -94,35 +84,25 @@ private predicate has_more_specific_dependency_source(Expr e) {
}
class PythonInheritance extends DependencyKind {
PythonInheritance() {
this = "inheritance"
}
PythonInheritance() { this = "inheritance" }
override predicate isADependency(AstNode source, Object target) {
this = this and
exists(ClassObject cls |
source = cls.getOrigin()
|
exists(ClassObject cls | source = cls.getOrigin() |
target = cls.getASuperType()
or
target = cls.getAnInferredType()
)
}
}
class PythonAttribute extends DependencyKind {
PythonAttribute() {
this = "attribute"
}
PythonAttribute() { this = "attribute" }
override predicate isADependency(AstNode source, Object target) {
this = this and
attribute_access_dependency(target, source)
}
}
private predicate attribute_access_dependency(Object target, AstNode source) {
@@ -133,30 +113,23 @@ private predicate attribute_access_dependency(Object target, AstNode source) {
}
private predicate use_of_attribute(Attribute attr, Scope s, string name) {
exists(AttrNode cfg |
cfg.isLoad() and cfg.getNode() = attr
|
exists(Object obj |
cfg.getObject(name).refersTo(obj) |
exists(AttrNode cfg | cfg.isLoad() and cfg.getNode() = attr |
exists(Object obj | cfg.getObject(name).refersTo(obj) |
s = obj.(PythonModuleObject).getModule() or
s = obj.(ClassObject).getPyClass()
) or
exists(ClassObject cls |
cfg.getObject(name).refersTo(_, cls, _) |
s = cls.getPyClass()
)
or
exists(ClassObject cls | cfg.getObject(name).refersTo(_, cls, _) | s = cls.getPyClass())
)
or
exists(SelfAttributeRead sar |
sar = attr |
exists(SelfAttributeRead sar | sar = attr |
sar.getClass() = s and
sar.getName() = name
)
}
private predicate defn_of_attribute(Object target, Scope s, string name) {
exists(Assign asgn |
target.(ControlFlowNode).getNode() = asgn |
exists(Assign asgn | target.(ControlFlowNode).getNode() = asgn |
defn_of_instance_attribute(asgn, s, name)
or
defn_of_class_attribute(asgn, s, name)
@@ -165,13 +138,14 @@ private predicate defn_of_attribute(Object target, Scope s, string name) {
defn_of_module_attribute(target, s, name)
}
/* Whether asgn defines an instance attribute, that is does
/*
* Whether asgn defines an instance attribute, that is does
* asgn take the form self.name = ... where self is an instance
* of class c and asgn is not a redefinition.
*/
private predicate defn_of_instance_attribute(Assign asgn, Class c, string name) {
exists(SelfAttributeStore sas |
asgn.getATarget() = sas |
exists(SelfAttributeStore sas | asgn.getATarget() = sas |
sas.getClass() = c and
sas.getName() = name and
not exists(SelfAttributeStore in_init |

20
python/ql/src/semmle/python/dependencies/DependencyKind.qll
View File

@@ -3,7 +3,6 @@ import semmle.python.dependencies.Dependencies
/**
* A library describing an abstract mechanism for representing dependency categories.
*/
/*
* A DependencyCategory is a unique string key used by Architect to identify different categories
* of dependencies that might be viewed independently.
@@ -12,20 +11,17 @@ import semmle.python.dependencies.Dependencies
* accepted by Architect.
* </p>
*/
abstract class DependencyKind extends string {
abstract class DependencyKind extends string {
bindingset[this]
DependencyKind() {
this = this
}
DependencyKind() { this = this }
/* Tech inventory interface */
/**
* Identify dependencies associated with this category.
* <p>
* The source element is the source of the dependency.
* </p>
*/
* Identify dependencies associated with this category.
* <p>
* The source element is the source of the dependency.
* </p>
*/
abstract predicate isADependency(AstNode source, Object target);
}
}

70
python/ql/src/semmle/python/dependencies/TechInventory.qll
View File

@@ -2,49 +2,41 @@ import python
import semmle.python.dependencies.Dependencies
import semmle.python.dependencies.DependencyKind
/** Combine the source-file and package into a single string:
/**
* Combine the source-file and package into a single string:
* /path/to/file.py<|>package-name-and-version
*/
string munge(File sourceFile, ExternalPackage package) {
result = "/" + sourceFile.getRelativePath() + "<|>" + package.getName() + "<|>" + package.getVersion() or
not exists(package.getVersion()) and result = "/" + sourceFile.getRelativePath() + "<|>" + package.getName() + "<|>unknown"
result =
"/" + sourceFile.getRelativePath() + "<|>" + package.getName() + "<|>" + package.getVersion()
or
not exists(package.getVersion()) and
result = "/" + sourceFile.getRelativePath() + "<|>" + package.getName() + "<|>unknown"
}
abstract class ExternalPackage extends Object {
ExternalPackage() {
this instanceof ModuleObject
}
ExternalPackage() { this instanceof ModuleObject }
abstract string getName();
abstract string getVersion();
Object getAttribute(string name) {
result = this.(ModuleObject).attr(name)
}
PackageObject getPackage() {
result = this.(ModuleObject).getPackage()
}
Object getAttribute(string name) { result = this.(ModuleObject).attr(name) }
PackageObject getPackage() { result = this.(ModuleObject).getPackage() }
}
bindingset[text]
private predicate is_version(string text) {
text.regexpMatch("\\d+\\.\\d+(\\.\\d+)?([ab]\\d+)?")
}
private predicate is_version(string text) { text.regexpMatch("\\d+\\.\\d+(\\.\\d+)?([ab]\\d+)?") }
bindingset[v]
private string version_format(float v) {
exists(int i, int f |
i = (v+0.05).floor() and f = ((v+0.05-i)*10).floor() |
exists(int i, int f | i = (v + 0.05).floor() and f = ((v + 0.05 - i) * 10).floor() |
result = i + "." + f
)
}
class DistPackage extends ExternalPackage {
DistPackage() {
exists(Folder parent |
parent = this.(ModuleObject).getPath().getParent() and
@@ -56,12 +48,13 @@ class DistPackage extends ExternalPackage {
)
}
/* We don't extract the meta-data for dependencies (yet), so make a best guess from the source
* https://www.python.org/dev/peps/pep-0396/
/*
* We don't extract the meta-data for dependencies (yet), so make a best guess from the source
* https://www.python.org/dev/peps/pep-0396/
*/
private predicate possibleVersion(string version, int priority) {
exists(Object v |
v = this.getAttribute("__version__") and priority = 3 |
exists(Object v | v = this.getAttribute("__version__") and priority = 3 |
version = v.(StringObject).getText() and is_version(version)
or
version = version_format(v.(NumericObject).floatValue())
@@ -71,33 +64,36 @@ class DistPackage extends ExternalPackage {
or
exists(SequenceObject tuple, NumericObject major, NumericObject minor, string base_version |
this.getAttribute("version_info") = tuple and
major = tuple.getInferredElement(0) and minor = tuple.getInferredElement(1) and
base_version = major.intValue() + "." + minor.intValue() |
major = tuple.getInferredElement(0) and
minor = tuple.getInferredElement(1) and
base_version = major.intValue() + "." + minor.intValue()
|
version = base_version + "." + tuple.getBuiltinElement(2).(NumericObject).intValue()
or
not exists(tuple.getBuiltinElement(2)) and version = base_version
) and priority = 2
) and
priority = 2
or
exists(string v |
v.toLowerCase() = "version" |
exists(string v | v.toLowerCase() = "version" |
is_version(version) and
version = this.getAttribute(v).(StringObject).getText()
) and priority = 1
) and
priority = 1
}
override string getVersion() {
this.possibleVersion(result, max(int priority | this.possibleVersion(_, priority)))
}
override string getName() {
result = this.(ModuleObject).getShortName()
}
override string getName() { result = this.(ModuleObject).getShortName() }
predicate fromSource(Object src) {
exists(ModuleObject m |
m.getModule() = src.(ControlFlowNode).getEnclosingModule() or
src = m |
m = this or
src = m
|
m = this
or
m.getPackage+() = this and
not exists(DistPackage inter |
m.getPackage*() = inter and
@@ -105,12 +101,10 @@ class DistPackage extends ExternalPackage {
)
)
}
}
predicate dependency(AstNode src, DistPackage package) {
exists(DependencyKind cat, Object target |
cat.isADependency(src, target) |
exists(DependencyKind cat, Object target | cat.isADependency(src, target) |
package.fromSource(target)
)
}

134
python/ql/src/semmle/python/essa/Definitions.qll
View File

@@ -1,6 +1,7 @@
import python
/* Classification of variables. These should be non-overlapping and complete.
/*
* Classification of variables. These should be non-overlapping and complete.
*
* Function local variables - Non escaping variables in a function, except 'self'
* Self variables - The 'self' variable for a method.
@@ -11,31 +12,24 @@ import python
* Escaping globals -- Global variables that have definitions and at least one of those definitions is in another scope.
*/
/** A source language variable, to be converted into a set of SSA variables. */
/** A source language variable, to be converted into a set of SSA variables. */
abstract class SsaSourceVariable extends @py_variable {
SsaSourceVariable() {
/* Exclude `True`, `False` and `None` */
not this.(Variable).getALoad() instanceof NameConstant
}
/** Gets the name of this variable */
string getName() {
variable(this, _, result)
}
string getName() { variable(this, _, result) }
Scope getScope() {
variable(this, result, _)
}
Scope getScope() { variable(this, result, _) }
/** Gets an implicit use of this variable */
abstract ControlFlowNode getAnImplicitUse();
abstract ControlFlowNode getScopeEntryDefinition();
string toString() {
result = "SsaSourceVariable " + this.getName()
}
string toString() { result = "SsaSourceVariable " + this.getName() }
/** Gets a use of this variable, either explicit or implicit. */
ControlFlowNode getAUse() {
@@ -43,14 +37,18 @@ abstract class SsaSourceVariable extends @py_variable {
or
result = this.getAnImplicitUse()
or
/* `import *` is a definition of *all* variables, so must be a use as well, for pass-through
/*
* `import *` is a definition of *all* variables, so must be a use as well, for pass-through
* once we have established that a variable is not redefined.
*/
SsaSource::import_star_refinement(this, result, _)
or
/* Add a use at the end of scope for all variables to keep them live
/*
* Add a use at the end of scope for all variables to keep them live
* This is necessary for taint-tracking.
*/
result = this.getScope().getANormalExit()
}
@@ -73,15 +71,18 @@ abstract class SsaSourceVariable extends @py_variable {
SsaSource::with_definition(this, def)
}
/** Holds if `def` defines an ESSA variable for this variable in such a way
/**
* Holds if `def` defines an ESSA variable for this variable in such a way
* that the new variable is a refinement in some way of the variable used at `use`.
*/
predicate hasRefinement(ControlFlowNode use, ControlFlowNode def) {
this.hasDefiningNode(_) and /* Can't have a refinement unless there is a definition */
this.hasDefiningNode(_) and
/* Can't have a refinement unless there is a definition */
refinement(this, use, def)
}
/** Holds if the edge `pred`->`succ` defines an ESSA variable for this variable in such a way
/**
* Holds if the edge `pred`->`succ` defines an ESSA variable for this variable in such a way
* that the new variable is a refinement in some way of the variable used at `use`.
*/
predicate hasRefinementEdge(ControlFlowNode use, BasicBlock pred, BasicBlock succ) {
@@ -100,7 +101,6 @@ abstract class SsaSourceVariable extends @py_variable {
}
abstract CallNode redefinedAtCallSite();
}
private predicate refinement(SsaSourceVariable v, ControlFlowNode use, ControlFlowNode def) {
@@ -119,9 +119,7 @@ private predicate refinement(SsaSourceVariable v, ControlFlowNode use, ControlFl
def = v.redefinedAtCallSite() and def = use
}
class FunctionLocalVariable extends SsaSourceVariable {
FunctionLocalVariable() {
this.(LocalVariable).getScope() instanceof Function and
not this instanceof NonLocalVariable
@@ -132,8 +130,7 @@ class FunctionLocalVariable extends SsaSourceVariable {
}
override ControlFlowNode getScopeEntryDefinition() {
exists(Scope s |
s.getEntryNode() = result |
exists(Scope s | s.getEntryNode() = result |
s = this.(LocalVariable).getScope() and
not this.(LocalVariable).isParameter()
or
@@ -143,11 +140,9 @@ class FunctionLocalVariable extends SsaSourceVariable {
}
override CallNode redefinedAtCallSite() { none() }
}
class NonLocalVariable extends SsaSourceVariable {
NonLocalVariable() {
exists(Function f |
this.(LocalVariable).getScope() = f and
@@ -169,37 +164,27 @@ class NonLocalVariable extends SsaSourceVariable {
this.(LocalVariable).getScope().getEntryNode() = result
}
pragma [noinline]
Scope scope_as_local_variable() {
result = this.(LocalVariable).getScope()
}
pragma[noinline]
Scope scope_as_local_variable() { result = this.(LocalVariable).getScope() }
override CallNode redefinedAtCallSite() {
result.getScope().getScope*() = this.scope_as_local_variable()
}
}
class ClassLocalVariable extends SsaSourceVariable {
ClassLocalVariable() { this.(LocalVariable).getScope() instanceof Class }
ClassLocalVariable() {
this.(LocalVariable).getScope() instanceof Class
}
override ControlFlowNode getAnImplicitUse() {
none()
}
override ControlFlowNode getAnImplicitUse() { none() }
override ControlFlowNode getScopeEntryDefinition() {
result = this.(LocalVariable).getScope().getEntryNode()
}
override CallNode redefinedAtCallSite() { none() }
}
class BuiltinVariable extends SsaSourceVariable {
BuiltinVariable() {
this instanceof GlobalVariable and
not exists(this.(Variable).getAStore()) and
@@ -208,20 +193,14 @@ class BuiltinVariable extends SsaSourceVariable {
not exists(ImportStar is | is.getScope() = this.(Variable).getScope())
}
override ControlFlowNode getAnImplicitUse() {
none()
}
override ControlFlowNode getAnImplicitUse() { none() }
override ControlFlowNode getScopeEntryDefinition() {
none()
}
override ControlFlowNode getScopeEntryDefinition() { none() }
override CallNode redefinedAtCallSite() { none() }
}
class ModuleVariable extends SsaSourceVariable {
ModuleVariable() {
this instanceof GlobalVariable and
(
@@ -235,10 +214,8 @@ class ModuleVariable extends SsaSourceVariable {
)
}
pragma [noinline]
CallNode global_variable_callnode() {
result.getScope() = this.(GlobalVariable).getScope()
}
pragma[noinline]
CallNode global_variable_callnode() { result.getScope() = this.(GlobalVariable).getScope() }
pragma[noinline]
ImportMemberNode global_variable_import() {
@@ -251,8 +228,7 @@ class ModuleVariable extends SsaSourceVariable {
or
result = global_variable_import()
or
exists(ImportTimeScope scope |
scope.entryEdge(result, _) |
exists(ImportTimeScope scope | scope.entryEdge(result, _) |
this = scope.getOuterVariable(_) or
this.(Variable).getAUse().getScope() = scope
)
@@ -264,14 +240,14 @@ class ModuleVariable extends SsaSourceVariable {
)
or
exists(ImportTimeScope s |
result = s.getANormalExit() and this.(Variable).getScope() = s and
result = s.getANormalExit() and
this.(Variable).getScope() = s and
implicit_definition(this)
)
}
override ControlFlowNode getScopeEntryDefinition() {
exists(Scope s |
s.getEntryNode() = result |
exists(Scope s | s.getEntryNode() = result |
/* Module entry point */
this.(GlobalVariable).getScope() = s
or
@@ -282,31 +258,28 @@ class ModuleVariable extends SsaSourceVariable {
this.(GlobalVariable).getAUse().getScope() = s
)
or
exists(ImportTimeScope scope |
scope.entryEdge(_, result) |
exists(ImportTimeScope scope | scope.entryEdge(_, result) |
this = scope.getOuterVariable(_) or
this.(Variable).getAUse().getScope() = scope
)
}
override CallNode redefinedAtCallSite() { none() }
}
class NonEscapingGlobalVariable extends ModuleVariable {
NonEscapingGlobalVariable() {
this instanceof GlobalVariable and
exists(this.(Variable).getAStore()) and
not variable_or_attribute_defined_out_of_scope(this)
}
}
class EscapingGlobalVariable extends ModuleVariable {
EscapingGlobalVariable() {
this instanceof GlobalVariable and exists(this.(Variable).getAStore()) and variable_or_attribute_defined_out_of_scope(this)
this instanceof GlobalVariable and
exists(this.(Variable).getAStore()) and
variable_or_attribute_defined_out_of_scope(this)
}
override ControlFlowNode getAnImplicitUse() {
@@ -328,36 +301,25 @@ class EscapingGlobalVariable extends ModuleVariable {
result = this.innerScope().getEntryNode()
}
pragma [noinline]
Scope scope_as_global_variable() {
result = this.(GlobalVariable).getScope()
}
pragma[noinline]
Scope scope_as_global_variable() { result = this.(GlobalVariable).getScope() }
override CallNode redefinedAtCallSite() {
result.(CallNode).getScope().getScope*() = this.scope_as_global_variable()
}
}
class EscapingAssignmentGlobalVariable extends EscapingGlobalVariable {
EscapingAssignmentGlobalVariable() {
exists(NameNode n | n.defines(this) and not n.getScope() = this.getScope())
}
}
class SpecialSsaSourceVariable extends SsaSourceVariable {
SpecialSsaSourceVariable() {
variable(this, _, "*") or variable(this, _, "$")
}
SpecialSsaSourceVariable() { variable(this, _, "*") or variable(this, _, "$") }
override ControlFlowNode getAnImplicitUse() {
exists(ImportTimeScope s |
result = s.getANormalExit() and this.getScope() = s
)
exists(ImportTimeScope s | result = s.getANormalExit() and this.getScope() = s)
}
override ControlFlowNode getScopeEntryDefinition() {
@@ -365,31 +327,31 @@ class SpecialSsaSourceVariable extends SsaSourceVariable {
this.getScope().getEntryNode() = result
}
pragma [noinline]
Scope scope_as_global_variable() {
result = this.(GlobalVariable).getScope()
}
pragma[noinline]
Scope scope_as_global_variable() { result = this.(GlobalVariable).getScope() }
override CallNode redefinedAtCallSite() {
result.(CallNode).getScope().getScope*() = this.scope_as_global_variable()
}
}
/** Holds if this variable is implicitly defined */
private predicate implicit_definition(Variable v) {
v.getId() = "*" or v.getId() = "$"
or
v.getId() = "*" or
v.getId() = "$" or
exists(ImportStar is | is.getScope() = v.getScope())
}
private predicate variable_or_attribute_defined_out_of_scope(Variable v) {
exists(NameNode n | n.defines(v) and not n.getScope() = v.getScope())
or
exists(AttrNode a | a.isStore() and a.getObject() = v.getAUse() and not a.getScope() = v.getScope())
exists(AttrNode a |
a.isStore() and a.getObject() = v.getAUse() and not a.getScope() = v.getScope()
)
}
private predicate class_with_global_metaclass(Class cls, GlobalVariable metaclass) {
metaclass.getId() = "__metaclass__" and major_version() = 2 and
metaclass.getId() = "__metaclass__" and
major_version() = 2 and
cls.getEnclosingModule() = metaclass.getScope()
}

405
python/ql/src/semmle/python/essa/Essa.qll
View File

@@ -6,46 +6,35 @@ import python
private import SsaCompute
import semmle.python.essa.Definitions
/** An (enhanced) SSA variable derived from `SsaSourceVariable`. */
class EssaVariable extends TEssaDefinition {
/** Gets the (unique) definition of this variable. */
EssaDefinition getDefinition() {
this = result
}
EssaDefinition getDefinition() { this = result }
/** Gets a use of this variable, where a "use" is defined by
/**
* Gets a use of this variable, where a "use" is defined by
* `SsaSourceVariable.getAUse()`.
* Note that this differs from `EssaVariable.getASourceUse()`.
*/
ControlFlowNode getAUse() {
result = this.getDefinition().getAUse()
}
ControlFlowNode getAUse() { result = this.getDefinition().getAUse() }
/** Gets the source variable from which this variable is derived. */
SsaSourceVariable getSourceVariable() {
result = this.getDefinition().getSourceVariable()
}
SsaSourceVariable getSourceVariable() { result = this.getDefinition().getSourceVariable() }
/** Gets the name of this variable. */
string getName() {
result = this.getSourceVariable().getName()
}
string getName() { result = this.getSourceVariable().getName() }
string toString() {
result = "SSA variable " + this.getName()
}
string toString() { result = "SSA variable " + this.getName() }
/** Gets a string representation of this variable.
/**
* Gets a string representation of this variable.
* WARNING: The format of this may change and it may be very inefficient to compute.
* To used for debugging and testing only.
*/
string getRepresentation() {
result = this.getSourceVariable().getName() + "_" + var_rank(this)
}
string getRepresentation() { result = this.getSourceVariable().getName() + "_" + var_rank(this) }
/** Gets a use of this variable, where a "use" is defined by
/**
* Gets a use of this variable, where a "use" is defined by
* `SsaSourceVariable.getASourceUse()`.
* Note that this differs from `EssaVariable.getAUse()`.
*/
@@ -63,23 +52,21 @@ class EssaVariable extends TEssaDefinition {
}
/** Gets the scope of this variable. */
Scope getScope() {
result = this.getDefinition().getScope()
}
Scope getScope() { result = this.getDefinition().getScope() }
/** Holds if this the meta-variable for a scope.
/**
* Holds if this the meta-variable for a scope.
* This is used to attach attributes for undeclared variables implicitly
* defined by `from ... import *` and the like.
*/
predicate isMetaVariable() {
this.getName() = "$"
}
predicate isMetaVariable() { this.getName() = "$" }
}
/* Helper for location_string
/*
* Helper for location_string
* NOTE: This is Python specific, to make `getRepresentation()` portable will require further work.
*/
private int exception_handling(BasicBlock b) {
b.reachesExit() and result = 0
or
@@ -91,14 +78,15 @@ pragma[noinline]
private string location_string(EssaVariable v) {
exists(EssaDefinition def, BasicBlock b, int index, int line, int col |
def = v.getDefinition() and
(if b.getNode(0).isNormalExit() then
line = 100000 and col = 0
else
b.hasLocationInfo(_, line, col, _, _)
(
if b.getNode(0).isNormalExit()
then line = 100000 and col = 0
else b.hasLocationInfo(_, line, col, _, _)
) and
/* Add large numbers to values to prevent 1000 sorting before 99 */
result = (line + 100000) + ":" + (col*2 + 10000 + exception_handling(b)) + ":" + (index + 100003)
|
result =
(line + 100000) + ":" + (col * 2 + 10000 + exception_handling(b)) + ":" + (index + 100003)
|
def = TEssaNodeDefinition(_, b, index)
or
def = TEssaNodeRefinement(_, b, index)
@@ -119,37 +107,31 @@ private int var_rank(EssaVariable v) {
exists(int r, SsaSourceVariable var |
var = v.getSourceVariable() and
var_index(v) = rank[r](EssaVariable x | x.getSourceVariable() = var | var_index(x)) and
result = r-1
result = r - 1
)
}
/** Underlying IPA type for EssaDefinition and EssaVariable. */
private cached newtype TEssaDefinition =
cached
private newtype TEssaDefinition =
TEssaNodeDefinition(SsaSourceVariable v, BasicBlock b, int i) {
EssaDefinitions::variableDefinition(v, _, b, _, i)
}
or
} or
TEssaNodeRefinement(SsaSourceVariable v, BasicBlock b, int i) {
EssaDefinitions::variableRefinement(v, _, b, _, i)
}
or
} or
TEssaEdgeDefinition(SsaSourceVariable v, BasicBlock pred, BasicBlock succ) {
EssaDefinitions::piNode(v, pred, succ)
}
or
TPhiFunction(SsaSourceVariable v, BasicBlock b) {
EssaDefinitions::phiNode(v, b)
}
} or
TPhiFunction(SsaSourceVariable v, BasicBlock b) { EssaDefinitions::phiNode(v, b) }
/** Definition of an extended-SSA (ESSA) variable.
/**
* Definition of an extended-SSA (ESSA) variable.
* There is exactly one definition for each variable,
* and exactly one variable for each definition.
*/
abstract class EssaDefinition extends TEssaDefinition {
string toString() {
result = "EssaDefinition"
}
string toString() { result = "EssaDefinition" }
/** Gets the source variable for which this a definition, either explicit or implicit. */
abstract SsaSourceVariable getSourceVariable();
@@ -160,7 +142,8 @@ abstract class EssaDefinition extends TEssaDefinition {
/** Holds if this definition reaches the end of `b`. */
abstract predicate reachesEndOfBlock(BasicBlock b);
/** Gets the location of a control flow node that is indicative of this definition.
/**
* Gets the location of a control flow node that is indicative of this definition.
* Since definitions may occur on edges of the control flow graph, the given location may
* be imprecise.
* Distinct `EssaDefinitions` may return the same ControlFlowNode even for
@@ -168,29 +151,26 @@ abstract class EssaDefinition extends TEssaDefinition {
*/
abstract Location getLocation();
/** Gets a representation of this SSA definition for debugging purposes.
* Since this is primarily for debugging and testing, performance may be poor. */
/**
* Gets a representation of this SSA definition for debugging purposes.
* Since this is primarily for debugging and testing, performance may be poor.
*/
abstract string getRepresentation();
abstract Scope getScope();
EssaVariable getVariable() {
result.getDefinition() = this
}
EssaVariable getVariable() { result.getDefinition() = this }
abstract BasicBlock getBasicBlock();
}
/** An ESSA definition corresponding to an edge refinement of the underlying variable.
/**
* An ESSA definition corresponding to an edge refinement of the underlying variable.
* For example, the edges leaving a test on a variable both represent refinements of that
* variable. On one edge the test is true, on the other it is false.
*/
class EssaEdgeRefinement extends EssaDefinition, TEssaEdgeDefinition {
override string toString() {
result = "SSA filter definition"
}
override string toString() { result = "SSA filter definition" }
boolean getSense() {
this.getPredecessor().getATrueSuccessor() = this.getSuccessor() and result = true
@@ -198,19 +178,13 @@ class EssaEdgeRefinement extends EssaDefinition, TEssaEdgeDefinition {
this.getPredecessor().getAFalseSuccessor() = this.getSuccessor() and result = false
}
override SsaSourceVariable getSourceVariable() {
this = TEssaEdgeDefinition(result, _, _)
}
override SsaSourceVariable getSourceVariable() { this = TEssaEdgeDefinition(result, _, _) }
/** Gets the basic block preceding the edge on which this refinement occurs. */
BasicBlock getPredecessor() {
this = TEssaEdgeDefinition(_, result, _)
}
BasicBlock getPredecessor() { this = TEssaEdgeDefinition(_, result, _) }
/** Gets the basic block succeeding the edge on which this refinement occurs. */
BasicBlock getSuccessor() {
this = TEssaEdgeDefinition(_, _, result)
}
BasicBlock getSuccessor() { this = TEssaEdgeDefinition(_, _, result) }
override ControlFlowNode getAUse() {
SsaDefinitions::reachesUse(this.getSourceVariable(), this.getSuccessor(), piIndex(), result)
@@ -220,13 +194,11 @@ class EssaEdgeRefinement extends EssaDefinition, TEssaEdgeDefinition {
SsaDefinitions::reachesEndOfBlock(this.getSourceVariable(), this.getSuccessor(), piIndex(), b)
}
override Location getLocation() {
result = this.getSuccessor().getNode(0).getLocation()
}
override Location getLocation() { result = this.getSuccessor().getNode(0).getLocation() }
/** Gets the SSA variable to which this refinement applies. */
EssaVariable getInput() {
exists(SsaSourceVariable var , EssaDefinition def |
exists(SsaSourceVariable var, EssaDefinition def |
var = this.getSourceVariable() and
var = def.getSourceVariable() and
def.reachesEndOfBlock(this.getPredecessor()) and
@@ -239,19 +211,13 @@ class EssaEdgeRefinement extends EssaDefinition, TEssaEdgeDefinition {
}
/** Gets the scope of the variable defined by this definition. */
override Scope getScope() {
result = this.getPredecessor().getScope()
}
override BasicBlock getBasicBlock(){
result = this.getSuccessor()
}
override Scope getScope() { result = this.getPredecessor().getScope() }
override BasicBlock getBasicBlock() { result = this.getSuccessor() }
}
/** A Phi-function as specified in classic SSA form. */
class PhiFunction extends EssaDefinition, TPhiFunction {
override ControlFlowNode getAUse() {
SsaDefinitions::reachesUse(this.getSourceVariable(), this.getBasicBlock(), phiIndex(), result)
}
@@ -260,9 +226,7 @@ class PhiFunction extends EssaDefinition, TPhiFunction {
SsaDefinitions::reachesEndOfBlock(this.getSourceVariable(), this.getBasicBlock(), phiIndex(), b)
}
override SsaSourceVariable getSourceVariable() {
this = TPhiFunction(result, _)
}
override SsaSourceVariable getSourceVariable() { this = TPhiFunction(result, _) }
/** Gets an input refinement that exists on one of the incoming edges to this phi node. */
private EssaEdgeRefinement inputEdgeRefinement(BasicBlock pred) {
@@ -290,37 +254,29 @@ class PhiFunction extends EssaDefinition, TPhiFunction {
}
/** Gets the input variable for this phi node on the edge `pred` -> `this.getBasicBlock()`, if any. */
cached EssaVariable getInput(BasicBlock pred) {
cached
EssaVariable getInput(BasicBlock pred) {
result.getDefinition() = this.reachingDefinition(pred)
or
result.getDefinition() = this.inputEdgeRefinement(pred)
}
/** Gets an input variable for this phi node. */
EssaVariable getAnInput() {
result = this.getInput(_)
}
EssaVariable getAnInput() { result = this.getInput(_) }
/** Holds if forall incoming edges in the flow graph, there is an input variable */
predicate isComplete() {
forall(BasicBlock pred |
pred = this.getBasicBlock().getAPredecessor() |
forall(BasicBlock pred | pred = this.getBasicBlock().getAPredecessor() |
exists(this.getInput(pred))
)
}
override string toString() {
result = "SSA Phi Function"
}
override string toString() { result = "SSA Phi Function" }
/** Gets the basic block that succeeds this phi node. */
override BasicBlock getBasicBlock() {
this = TPhiFunction(_, result)
}
override BasicBlock getBasicBlock() { this = TPhiFunction(_, result) }
override Location getLocation() {
result = this.getBasicBlock().getNode(0).getLocation()
}
override Location getLocation() { result = this.getBasicBlock().getNode(0).getLocation() }
/** Helper for `argList(n)`. */
private int rankInput(EssaVariable input) {
@@ -330,12 +286,10 @@ class PhiFunction extends EssaDefinition, TPhiFunction {
/** Helper for `argList()`. */
private string argList(int n) {
exists(EssaVariable input |
n = this.rankInput(input)
|
exists(EssaVariable input | n = this.rankInput(input) |
n = 1 and result = input.getRepresentation()
or
n > 1 and result = this.argList(n-1) + ", " + input.getRepresentation()
n > 1 and result = this.argList(n - 1) + ", " + input.getRepresentation()
)
}
@@ -352,13 +306,12 @@ class PhiFunction extends EssaDefinition, TPhiFunction {
or
result = "phi(" + this.argList() + ")"
or
exists(this.getAnInput()) and not exists(this.argList()) and
exists(this.getAnInput()) and
not exists(this.argList()) and
result = "phi(" + this.getSourceVariable().getName() + "??)"
}
override Scope getScope() {
result = this.getBasicBlock().getScope()
}
override Scope getScope() { result = this.getBasicBlock().getScope() }
private EssaEdgeRefinement piInputDefinition(EssaVariable input) {
input = this.getAnInput() and
@@ -367,7 +320,8 @@ class PhiFunction extends EssaDefinition, TPhiFunction {
input = this.getAnInput() and result = input.getDefinition().(PhiFunction).piInputDefinition(_)
}
/** Gets the variable which is the common and complete input to all pi-nodes that are themselves
/**
* Gets the variable which is the common and complete input to all pi-nodes that are themselves
* inputs to this phi-node.
* For example:
* ```
@@ -387,19 +341,15 @@ class PhiFunction extends EssaDefinition, TPhiFunction {
* meaning that we cannot track `x` from `x0` to `x3`.
* By using `getShortCircuitInput()` we can do so, since the short-circuit input of `x3` is `x0`.
*/
pragma [noinline]
pragma[noinline]
EssaVariable getShortCircuitInput() {
exists(BasicBlock common |
forall(EssaVariable input |
input = this.getAnInput() |
forall(EssaVariable input | input = this.getAnInput() |
common = this.piInputDefinition(input).getPredecessor()
)
and
forall(BasicBlock succ |
succ = common.getASuccessor() |
) and
forall(BasicBlock succ | succ = common.getASuccessor() |
succ = this.piInputDefinition(_).getSuccessor()
)
and
) and
exists(EssaEdgeRefinement ref |
ref = this.piInputDefinition(_) and
ref.getPredecessor() = common and
@@ -409,14 +359,12 @@ class PhiFunction extends EssaDefinition, TPhiFunction {
}
}
/** A definition of an ESSA variable that is not directly linked to
/**
* A definition of an ESSA variable that is not directly linked to
* another ESSA variable.
*/
class EssaNodeDefinition extends EssaDefinition, TEssaNodeDefinition {
override string toString() {
result = "Essa node definition"
}
override string toString() { result = "Essa node definition" }
override ControlFlowNode getAUse() {
exists(SsaSourceVariable v, BasicBlock b, int i |
@@ -432,22 +380,14 @@ class EssaNodeDefinition extends EssaDefinition, TEssaNodeDefinition {
)
}
override SsaSourceVariable getSourceVariable() {
this = TEssaNodeDefinition(result, _, _)
}
override SsaSourceVariable getSourceVariable() { this = TEssaNodeDefinition(result, _, _) }
/** Gets the ControlFlowNode corresponding to this definition */
ControlFlowNode getDefiningNode() {
this.definedBy(_, result)
}
ControlFlowNode getDefiningNode() { this.definedBy(_, result) }
override Location getLocation() {
result = this.getDefiningNode().getLocation()
}
override Location getLocation() { result = this.getDefiningNode().getLocation() }
override string getRepresentation() {
result = this.getAQlClass()
}
override string getRepresentation() { result = this.getAQlClass() }
override Scope getScope() {
exists(BasicBlock defb |
@@ -457,26 +397,19 @@ class EssaNodeDefinition extends EssaDefinition, TEssaNodeDefinition {
}
predicate definedBy(SsaSourceVariable v, ControlFlowNode def) {
exists(BasicBlock b, int i |
def = b.getNode(i) |
this = TEssaNodeDefinition(v, b, i+i)
exists(BasicBlock b, int i | def = b.getNode(i) |
this = TEssaNodeDefinition(v, b, i + i)
or
this = TEssaNodeDefinition(v, b, i+i+1)
this = TEssaNodeDefinition(v, b, i + i + 1)
)
}
override BasicBlock getBasicBlock(){
result = this.getDefiningNode().getBasicBlock()
}
override BasicBlock getBasicBlock() { result = this.getDefiningNode().getBasicBlock() }
}
/** A definition of an ESSA variable that takes another ESSA variable as an input. */
class EssaNodeRefinement extends EssaDefinition, TEssaNodeRefinement {
override string toString() {
result = "SSA filter definition"
}
override string toString() { result = "SSA filter definition" }
/** Gets the SSA variable to which this refinement applies. */
EssaVariable getInput() {
@@ -498,18 +431,12 @@ class EssaNodeRefinement extends EssaDefinition, TEssaNodeRefinement {
)
}
override SsaSourceVariable getSourceVariable() {
this = TEssaNodeRefinement(result, _, _)
}
override SsaSourceVariable getSourceVariable() { this = TEssaNodeRefinement(result, _, _) }
/** Gets the ControlFlowNode corresponding to this definition */
ControlFlowNode getDefiningNode() {
this.definedBy(_, result)
}
ControlFlowNode getDefiningNode() { this.definedBy(_, result) }
override Location getLocation() {
result = this.getDefiningNode().getLocation()
}
override Location getLocation() { result = this.getDefiningNode().getLocation() }
override string getRepresentation() {
result = this.getAQlClass() + "(" + this.getInput().getRepresentation() + ")"
@@ -526,23 +453,19 @@ class EssaNodeRefinement extends EssaDefinition, TEssaNodeRefinement {
}
predicate definedBy(SsaSourceVariable v, ControlFlowNode def) {
exists(BasicBlock b, int i |
def = b.getNode(i) |
this = TEssaNodeRefinement(v, b, i+i)
exists(BasicBlock b, int i | def = b.getNode(i) |
this = TEssaNodeRefinement(v, b, i + i)
or
this = TEssaNodeRefinement(v, b, i+i+1)
this = TEssaNodeRefinement(v, b, i + i + 1)
)
}
override BasicBlock getBasicBlock(){
result = this.getDefiningNode().getBasicBlock()
}
override BasicBlock getBasicBlock() { result = this.getDefiningNode().getBasicBlock() }
}
pragma[noopt]
private EssaVariable potential_input(EssaNodeRefinement ref) {
exists(ControlFlowNode use, SsaSourceVariable var, ControlFlowNode def |
exists(ControlFlowNode use, SsaSourceVariable var, ControlFlowNode def |
var.hasRefinement(use, def) and
use = result.getAUse() and
var = result.getSourceVariable() and
@@ -559,7 +482,6 @@ deprecated class PyNodeRefinement = EssaNodeRefinement;
/** An assignment to a variable `v = val` */
class AssignmentDefinition extends EssaNodeDefinition {
AssignmentDefinition() {
SsaSource::assignment_definition(this.getSourceVariable(), this.getDefiningNode(), _)
}
@@ -568,15 +490,11 @@ class AssignmentDefinition extends EssaNodeDefinition {
SsaSource::assignment_definition(this.getSourceVariable(), this.getDefiningNode(), result)
}
override string getRepresentation() {
result = this.getValue().getNode().toString()
}
override string getRepresentation() { result = this.getValue().getNode().toString() }
}
/** Capture of a raised exception `except ExceptionType ex:` */
class ExceptionCapture extends EssaNodeDefinition {
class ExceptionCapture extends EssaNodeDefinition {
ExceptionCapture() {
SsaSource::exception_capture(this.getSourceVariable(), this.getDefiningNode())
}
@@ -588,15 +506,11 @@ class ExceptionCapture extends EssaNodeDefinition {
)
}
override string getRepresentation() {
result = "except " + this.getSourceVariable().getName()
}
override string getRepresentation() { result = "except " + this.getSourceVariable().getName() }
}
/** An assignment to a variable as part of a multiple assignment `..., v, ... = val` */
class MultiAssignmentDefinition extends EssaNodeDefinition {
MultiAssignmentDefinition() {
SsaSource::multi_assignment_definition(this.getSourceVariable(), this.getDefiningNode(), _, _)
}
@@ -610,130 +524,99 @@ class MultiAssignmentDefinition extends EssaNodeDefinition {
/** Holds if `this` has (zero-based) index `index` in `lhs`. */
predicate indexOf(int index, SequenceNode lhs) {
SsaSource::multi_assignment_definition(this.getSourceVariable(), this.getDefiningNode(), index, lhs)
SsaSource::multi_assignment_definition(this.getSourceVariable(), this.getDefiningNode(), index,
lhs)
}
}
/** A definition of a variable in a `with` statement */
class WithDefinition extends EssaNodeDefinition {
WithDefinition() { SsaSource::with_definition(this.getSourceVariable(), this.getDefiningNode()) }
WithDefinition () {
SsaSource::with_definition(this.getSourceVariable(), this.getDefiningNode())
}
override string getRepresentation() {
result = "with"
}
override string getRepresentation() { result = "with" }
}
/** A definition of a variable by declaring it as a parameter */
class ParameterDefinition extends EssaNodeDefinition {
ParameterDefinition() {
SsaSource::parameter_definition(this.getSourceVariable(), this.getDefiningNode())
}
predicate isSelf() {
this.getDefiningNode().getNode().(Parameter).isSelf()
}
predicate isSelf() { this.getDefiningNode().getNode().(Parameter).isSelf() }
/** Gets the control flow node for the default value of this parameter */
ControlFlowNode getDefault() {
result.getNode() = this.getParameter().getDefault()
}
ControlFlowNode getDefault() { result.getNode() = this.getParameter().getDefault() }
/** Gets the annotation control flow node of this parameter */
ControlFlowNode getAnnotation() {
result.getNode() = this.getParameter().getAnnotation()
}
ControlFlowNode getAnnotation() { result.getNode() = this.getParameter().getAnnotation() }
/** Gets the name of this parameter definition */
string getName() {
result = this.getParameter().asName().getId()
}
string getName() { result = this.getParameter().asName().getId() }
predicate isVarargs() {
exists(Function func | func.getVararg() = this.getDefiningNode().getNode())
}
/** Holds if this parameter is a 'kwargs' parameter.
/**
* Holds if this parameter is a 'kwargs' parameter.
* The `kwargs` in `f(a, b, **kwargs)`.
*/
predicate isKwargs() {
exists(Function func | func.getKwarg() = this.getDefiningNode().getNode())
}
Parameter getParameter() {
result = this.getDefiningNode().getNode()
}
Parameter getParameter() { result = this.getDefiningNode().getNode() }
}
/** A deletion of a variable `del v` */
class DeletionDefinition extends EssaNodeDefinition {
DeletionDefinition() {
SsaSource::deletion_definition(this.getSourceVariable(), this.getDefiningNode())
}
}
/** Definition of variable at the entry of a scope. Usually this represents the transfer of
/**
* Definition of variable at the entry of a scope. Usually this represents the transfer of
* a global or non-local variable from one scope to another.
*/
class ScopeEntryDefinition extends EssaNodeDefinition {
ScopeEntryDefinition() {
this.getDefiningNode() = this.getSourceVariable().getScopeEntryDefinition() and
not this instanceof ImplicitSubModuleDefinition
}
override Scope getScope() {
result.getEntryNode() = this.getDefiningNode()
}
override Scope getScope() { result.getEntryNode() = this.getDefiningNode() }
}
/** Possible redefinition of variable via `from ... import *` */
class ImportStarRefinement extends EssaNodeRefinement {
ImportStarRefinement() {
SsaSource::import_star_refinement(this.getSourceVariable(), _, this.getDefiningNode())
}
}
/** Assignment of an attribute `obj.attr = val` */
class AttributeAssignment extends EssaNodeRefinement {
AttributeAssignment() {
SsaSource::attribute_assignment_refinement(this.getSourceVariable(), _, this.getDefiningNode())
}
string getName() {
result = this.getDefiningNode().(AttrNode).getName()
}
string getName() { result = this.getDefiningNode().(AttrNode).getName() }
ControlFlowNode getValue() {
result = this.getDefiningNode().(DefinitionNode).getValue()
}
ControlFlowNode getValue() { result = this.getDefiningNode().(DefinitionNode).getValue() }
override string getRepresentation() {
result = this.getAQlClass() + " '" + this.getName() + "'(" + this.getInput().getRepresentation() + ")"
result =
this.getAQlClass() + " '" + this.getName() + "'(" + this.getInput().getRepresentation() + ")"
or
not exists(this.getInput()) and
result = this.getAQlClass() + " '" + this.getName() + "'(" + this.getSourceVariable().getName() + "??)"
result =
this.getAQlClass() + " '" + this.getName() + "'(" + this.getSourceVariable().getName() + "??)"
}
}
/** A use of a variable as an argument, `foo(v)`, which might modify the object referred to. */
class ArgumentRefinement extends EssaNodeRefinement {
ControlFlowNode argument;
ArgumentRefinement() {
@@ -747,20 +630,15 @@ class ArgumentRefinement extends EssaNodeRefinement {
/** Deletion of an attribute `del obj.attr`. */
class EssaAttributeDeletion extends EssaNodeRefinement {
EssaAttributeDeletion() {
SsaSource::attribute_deletion_refinement(this.getSourceVariable(), _, this.getDefiningNode())
}
string getName() {
result = this.getDefiningNode().(AttrNode).getName()
}
string getName() { result = this.getDefiningNode().(AttrNode).getName() }
}
/** A pi-node (guard) with only one successor. */
class SingleSuccessorGuard extends EssaNodeRefinement {
SingleSuccessorGuard() {
SsaSource::test_refinement(this.getSourceVariable(), _, this.getDefiningNode())
}
@@ -778,35 +656,28 @@ class SingleSuccessorGuard extends EssaNodeRefinement {
result = EssaNodeRefinement.super.getRepresentation() + " [??]"
}
ControlFlowNode getTest() {
result = this.getDefiningNode()
}
ControlFlowNode getTest() { result = this.getDefiningNode() }
predicate useAndTest(ControlFlowNode use, ControlFlowNode test) {
test = this.getDefiningNode() and
SsaSource::test_refinement(this.getSourceVariable(), use, test)
}
}
/** Implicit definition of the names of sub-modules in a package.
/**
* Implicit definition of the names of sub-modules in a package.
* Although the interpreter does not pre-define these names, merely populating them
* as they are imported, this is a good approximation for static analysis.
*/
class ImplicitSubModuleDefinition extends EssaNodeDefinition {
ImplicitSubModuleDefinition() {
SsaSource::init_module_submodule_defn(this.getSourceVariable(), this.getDefiningNode())
}
}
/** An implicit (possible) definition of an escaping variable at a call-site */
class CallsiteRefinement extends EssaNodeRefinement {
override string toString() {
result = "CallsiteRefinement"
}
override string toString() { result = "CallsiteRefinement" }
CallsiteRefinement() {
exists(SsaSourceVariable var, ControlFlowNode defn |
@@ -818,49 +689,37 @@ class CallsiteRefinement extends EssaNodeRefinement {
)
}
CallNode getCall() {
this.getDefiningNode() = result
}
CallNode getCall() { this.getDefiningNode() = result }
}
/** An implicit (possible) modification of the object referred at a method call */
class MethodCallsiteRefinement extends EssaNodeRefinement {
MethodCallsiteRefinement() {
SsaSource::method_call_refinement(this.getSourceVariable(), _, this.getDefiningNode())
and not this instanceof SingleSuccessorGuard
}
CallNode getCall() {
this.getDefiningNode() = result
SsaSource::method_call_refinement(this.getSourceVariable(), _, this.getDefiningNode()) and
not this instanceof SingleSuccessorGuard
}
CallNode getCall() { this.getDefiningNode() = result }
}
/** An implicit (possible) modification of `self` at a method call */
class SelfCallsiteRefinement extends MethodCallsiteRefinement {
SelfCallsiteRefinement() {
this.getSourceVariable().(Variable).isSelf()
}
SelfCallsiteRefinement() { this.getSourceVariable().(Variable).isSelf() }
}
/** Python specific sub-class of generic EssaEdgeRefinement */
class PyEdgeRefinement extends EssaEdgeRefinement {
override string getRepresentation() {
/* This is for testing so use capital 'P' to make it sort before 'phi' and
* be more visually distinctive. */
/*
* This is for testing so use capital 'P' to make it sort before 'phi' and
* be more visually distinctive.
*/
result = "Pi(" + this.getInput().getRepresentation() + ") [" + this.getSense() + "]"
or
not exists(this.getInput()) and
result = "Pi(" + this.getSourceVariable().getName() + "??) [" + this.getSense() + "]"
}
ControlFlowNode getTest() {
result = this.getPredecessor().getLastNode()
}
ControlFlowNode getTest() { result = this.getPredecessor().getLastNode() }
}

225
python/ql/src/semmle/python/essa/SsaCompute.qll
View File

@@ -1,28 +1,29 @@
/** Provides predicates for computing Enhanced SSA form
/**
* Provides predicates for computing Enhanced SSA form
* Computation of ESSA form is identical to plain SSA form,
* but what counts as a use of definition differs.
*
*
* ## Language independent data-flow graph construction
*
*
* Construction of the data-flow graph is based on the principles behind SSA variables.
*
*
* The definition of an SSA variable is that (statically):
*
*
* * Each variable has exactly one definition
* * A variable's definition dominates all its uses.
*
* SSA form was originally designed for compiler use and thus a "definition" of an SSA variable is
* the same as a definition of the underlying source-code variable. For register allocation this is
*
* SSA form was originally designed for compiler use and thus a "definition" of an SSA variable is
* the same as a definition of the underlying source-code variable. For register allocation this is
* sufficient to treat the variable as equivalent to the value held in the variable.
*
*
* However, this doesn't always work the way we want it for data-flow analysis.
*
* When we start to consider attribute assignment, tests on the value referred to be a variable,
*
* When we start to consider attribute assignment, tests on the value referred to be a variable,
* escaping variables, implicit definitions, etc., we need something finer grained.
*
*
* A data-flow variable has the same properties as a normal SSA variable, but it also has the property that
* *anything* that may change the way we view an object referred to by a variable should be treated as a definition of that variable.
*
*
* For example, tests are treated as definitions, so for the following Python code:
* ```python
* x = None
@@ -39,7 +40,7 @@
* from which is it possible to infer that `x3` may not be None.
* [ Phi functions are standard SSA, a Pi function is a filter or guard on the possible values that a variable
* may hold]
*
*
* Attribute assignments are also treated as definitions, so for the following Python code:
* ```python
* x = C()
@@ -55,13 +56,13 @@
* y1 = attr-assign(y0, .b = 1)
* ```
* From which we can infer that `x1.a` is `1` but we know nothing about `y0.a` despite it being the same type.
*
*
* We can also insert "definitions" for transfers of values (say in global variables) where we do not yet know the call-graph. For example,
* ```python
* def foo():
* global g
* g = 1
*
*
* def bar():
* foo()
* g
@@ -72,7 +73,7 @@
* def foo():
* g0 = scope-entry(g)
* g1 = 1
*
*
* def bar():
* g2 = scope-entry(g)
* foo()
@@ -86,57 +87,63 @@
* g3 = g1
* ```
* and thus it falls out that `g3` must be `1`.
*
*/
import python
private cached module SsaComputeImpl {
cached module EssaDefinitionsImpl {
cached
private module SsaComputeImpl {
cached
module EssaDefinitionsImpl {
/** Whether `n` is a live update that is a definition of the variable `v`. */
cached predicate variableDefinition(SsaSourceVariable v, ControlFlowNode n, BasicBlock b, int rankix, int i) {
cached
predicate variableDefinition(
SsaSourceVariable v, ControlFlowNode n, BasicBlock b, int rankix, int i
) {
SsaComputeImpl::variableDefine(v, n, b, i) and
SsaComputeImpl::defUseRank(v, b, rankix, i) and
(
SsaComputeImpl::defUseRank(v, b, rankix+1, _) and not SsaComputeImpl::defRank(v, b, rankix+1, _)
(
SsaComputeImpl::defUseRank(v, b, rankix + 1, _) and
not SsaComputeImpl::defRank(v, b, rankix + 1, _)
or
not SsaComputeImpl::defUseRank(v, b, rankix+1, _) and Liveness::liveAtExit(v, b)
not SsaComputeImpl::defUseRank(v, b, rankix + 1, _) and Liveness::liveAtExit(v, b)
)
}
/** Whether `n` is a live update that is a definition of the variable `v`. */
cached predicate variableRefinement(SsaSourceVariable v, ControlFlowNode n, BasicBlock b, int rankix, int i) {
cached
predicate variableRefinement(
SsaSourceVariable v, ControlFlowNode n, BasicBlock b, int rankix, int i
) {
SsaComputeImpl::variableRefine(v, n, b, i) and
SsaComputeImpl::defUseRank(v, b, rankix, i) and
(
SsaComputeImpl::defUseRank(v, b, rankix+1, _) and not SsaComputeImpl::defRank(v, b, rankix+1, _)
(
SsaComputeImpl::defUseRank(v, b, rankix + 1, _) and
not SsaComputeImpl::defRank(v, b, rankix + 1, _)
or
not SsaComputeImpl::defUseRank(v, b, rankix+1, _) and Liveness::liveAtExit(v, b)
not SsaComputeImpl::defUseRank(v, b, rankix + 1, _) and Liveness::liveAtExit(v, b)
)
}
cached predicate variableUpdate(SsaSourceVariable v, ControlFlowNode n, BasicBlock b, int rankix, int i) {
cached
predicate variableUpdate(SsaSourceVariable v, ControlFlowNode n, BasicBlock b, int rankix, int i) {
variableDefinition(v, n, b, rankix, i)
or
variableRefinement(v, n, b, rankix, i)
}
/** Holds if `def` is a pi-node for `v` on the edge `pred` -> `succ` */
cached predicate piNode(SsaSourceVariable v, BasicBlock pred, BasicBlock succ) {
cached
predicate piNode(SsaSourceVariable v, BasicBlock pred, BasicBlock succ) {
v.hasRefinementEdge(_, pred, succ) and
Liveness::liveAtEntry(v, succ)
}
/** A phi node for `v` at the beginning of basic block `b`. */
cached predicate phiNode(SsaSourceVariable v, BasicBlock b) {
/** A phi node for `v` at the beginning of basic block `b`. */
cached
predicate phiNode(SsaSourceVariable v, BasicBlock b) {
(
exists(BasicBlock def | def.dominanceFrontier(b) |
SsaComputeImpl::ssaDef(v, def)
)
exists(BasicBlock def | def.dominanceFrontier(b) | SsaComputeImpl::ssaDef(v, def))
or
piNode(v, _, b) and strictcount(b.getAPredecessor()) > 1
) and
@@ -144,25 +151,26 @@ private cached module SsaComputeImpl {
}
}
cached predicate variableDefine(SsaSourceVariable v, ControlFlowNode n, BasicBlock b, int i) {
v.hasDefiningNode(n)
and
cached
predicate variableDefine(SsaSourceVariable v, ControlFlowNode n, BasicBlock b, int i) {
v.hasDefiningNode(n) and
exists(int j |
n = b.getNode(j) and
i = j*2 + 1
i = j * 2 + 1
)
}
cached predicate variableRefine(SsaSourceVariable v, ControlFlowNode n, BasicBlock b, int i) {
v.hasRefinement(_, n)
and
cached
predicate variableRefine(SsaSourceVariable v, ControlFlowNode n, BasicBlock b, int i) {
v.hasRefinement(_, n) and
exists(int j |
n = b.getNode(j) and
i = j*2 + 1
i = j * 2 + 1
)
}
cached predicate variableDef(SsaSourceVariable v, ControlFlowNode n, BasicBlock b, int i) {
cached
predicate variableDef(SsaSourceVariable v, ControlFlowNode n, BasicBlock b, int i) {
variableDefine(v, n, b, i) or variableRefine(v, n, b, i)
}
@@ -174,22 +182,25 @@ private cached module SsaComputeImpl {
* irrelevant indices at which there is no definition or use when traversing
* basic blocks.
*/
cached predicate defUseRank(SsaSourceVariable v, BasicBlock b, int rankix, int i) {
i = rank[rankix](int j | variableDef(v, _, b, j) or variableUse(v, _, b, j))
cached
predicate defUseRank(SsaSourceVariable v, BasicBlock b, int rankix, int i) {
i = rank[rankix](int j | variableDef(v, _, b, j) or variableUse(v, _, b, j))
}
/** A definition of a variable occurring at the specified rank index in basic block `b`. */
cached predicate defRank(SsaSourceVariable v, BasicBlock b, int rankix, int i) {
variableDef(v, _, b, i) and
defUseRank(v, b, rankix, i)
cached
predicate defRank(SsaSourceVariable v, BasicBlock b, int rankix, int i) {
variableDef(v, _, b, i) and
defUseRank(v, b, rankix, i)
}
/** A `VarAccess` `use` of `v` in `b` at index `i`. */
cached predicate variableUse(SsaSourceVariable v, ControlFlowNode use, BasicBlock b, int i) {
cached
predicate variableUse(SsaSourceVariable v, ControlFlowNode use, BasicBlock b, int i) {
(v.getAUse() = use or v.hasRefinement(use, _)) and
exists(int j |
b.getNode(j) = use and
i = 2*j
b.getNode(j) = use and
i = 2 * j
)
}
@@ -197,7 +208,8 @@ private cached module SsaComputeImpl {
* A definition of an SSA variable occurring at the specified position.
* This is either a phi node, a `VariableUpdate`, or a parameter.
*/
cached predicate ssaDef(SsaSourceVariable v, BasicBlock b) {
cached
predicate ssaDef(SsaSourceVariable v, BasicBlock b) {
EssaDefinitions::phiNode(v, b)
or
EssaDefinitions::variableUpdate(v, _, b, _, _)
@@ -216,10 +228,13 @@ private cached module SsaComputeImpl {
*/
/** The maximum rank index for the given variable and basic block. */
cached int lastRank(SsaSourceVariable v, BasicBlock b) {
cached
int lastRank(SsaSourceVariable v, BasicBlock b) {
result = max(int rankix | defUseRank(v, b, rankix, _))
or
not defUseRank(v, b, _, _) and (EssaDefinitions::phiNode(v, b) or EssaDefinitions::piNode(v, _, b)) and result = 0
not defUseRank(v, b, _, _) and
(EssaDefinitions::phiNode(v, b) or EssaDefinitions::piNode(v, _, b)) and
result = 0
}
private predicate ssaDefRank(SsaSourceVariable v, BasicBlock b, int rankix, int i) {
@@ -227,22 +242,35 @@ private cached module SsaComputeImpl {
or
EssaDefinitions::phiNode(v, b) and rankix = 0 and i = phiIndex()
or
EssaDefinitions::piNode(v, _, b) and EssaDefinitions::phiNode(v, b) and rankix = -1 and i = piIndex()
EssaDefinitions::piNode(v, _, b) and
EssaDefinitions::phiNode(v, b) and
rankix = -1 and
i = piIndex()
or
EssaDefinitions::piNode(v, _, b) and not EssaDefinitions::phiNode(v, b) and rankix = 0 and i = piIndex()
EssaDefinitions::piNode(v, _, b) and
not EssaDefinitions::phiNode(v, b) and
rankix = 0 and
i = piIndex()
}
/** The SSA definition reaches the rank index `rankix` in its own basic block `b`. */
cached predicate ssaDefReachesRank(SsaSourceVariable v, BasicBlock b, int i, int rankix) {
ssaDefRank(v, b, rankix, i) or
ssaDefReachesRank(v, b, i, rankix-1) and rankix <= lastRank(v, b) and not ssaDefRank(v, b, rankix, _)
cached
predicate ssaDefReachesRank(SsaSourceVariable v, BasicBlock b, int i, int rankix) {
ssaDefRank(v, b, rankix, i)
or
ssaDefReachesRank(v, b, i, rankix - 1) and
rankix <= lastRank(v, b) and
not ssaDefRank(v, b, rankix, _)
}
/**
* The SSA definition of `v` at `def` reaches `use` in the same basic block
* without crossing another SSA definition of `v`.
*/
cached predicate ssaDefReachesUseWithinBlock(SsaSourceVariable v, BasicBlock b, int i, ControlFlowNode use) {
cached
predicate ssaDefReachesUseWithinBlock(
SsaSourceVariable v, BasicBlock b, int i, ControlFlowNode use
) {
exists(int rankix, int useix |
ssaDefReachesRank(v, b, i, rankix) and
defUseRank(v, b, rankix, useix) and
@@ -250,41 +278,44 @@ private cached module SsaComputeImpl {
)
}
cached module LivenessImpl {
cached
module LivenessImpl {
cached
predicate liveAtExit(SsaSourceVariable v, BasicBlock b) { liveAtEntry(v, b.getASuccessor()) }
cached predicate liveAtExit(SsaSourceVariable v, BasicBlock b) {
liveAtEntry(v, b.getASuccessor())
}
cached predicate liveAtEntry(SsaSourceVariable v, BasicBlock b) {
cached
predicate liveAtEntry(SsaSourceVariable v, BasicBlock b) {
SsaComputeImpl::defUseRank(v, b, 1, _) and not SsaComputeImpl::defRank(v, b, 1, _)
or
not SsaComputeImpl::defUseRank(v, b, _, _) and liveAtExit(v, b)
}
}
cached module SsaDefinitionsImpl {
pragma [noinline]
private predicate reachesEndOfBlockRec(SsaSourceVariable v, BasicBlock defbb, int defindex, BasicBlock b) {
cached
module SsaDefinitionsImpl {
pragma[noinline]
private predicate reachesEndOfBlockRec(
SsaSourceVariable v, BasicBlock defbb, int defindex, BasicBlock b
) {
exists(BasicBlock idom | reachesEndOfBlock(v, defbb, defindex, idom) |
idom = b.getImmediateDominator()
)
}
/**
* The SSA definition of `v` at `def` reaches the end of a basic block `b`, at
* which point it is still live, without crossing another SSA definition of `v`.
*/
cached
predicate reachesEndOfBlock(SsaSourceVariable v, BasicBlock defbb, int defindex, BasicBlock b) {
Liveness::liveAtExit(v, b) and
(
defbb = b and SsaComputeImpl::ssaDefReachesRank(v, defbb, defindex, SsaComputeImpl::lastRank(v, b))
or
// It is sufficient to traverse the dominator graph, cf. discussion above.
reachesEndOfBlockRec(v, defbb, defindex, b) and
not SsaComputeImpl::ssaDef(v, b)
Liveness::liveAtExit(v, b) and
(
defbb = b and
SsaComputeImpl::ssaDefReachesRank(v, defbb, defindex, SsaComputeImpl::lastRank(v, b))
or
// It is sufficient to traverse the dominator graph, cf. discussion above.
reachesEndOfBlockRec(v, defbb, defindex, b) and
not SsaComputeImpl::ssaDef(v, b)
)
}
@@ -294,15 +325,16 @@ private cached module SsaComputeImpl {
*/
cached
predicate reachesUse(SsaSourceVariable v, BasicBlock defbb, int defindex, ControlFlowNode use) {
SsaComputeImpl::ssaDefReachesUseWithinBlock(v, defbb, defindex, use) or
exists(BasicBlock b |
SsaComputeImpl::variableUse(v, use, b, _) and
reachesEndOfBlock(v, defbb, defindex, b.getAPredecessor()) and
not SsaComputeImpl::ssaDefReachesUseWithinBlock(v, b, _, use)
)
SsaComputeImpl::ssaDefReachesUseWithinBlock(v, defbb, defindex, use)
or
exists(BasicBlock b |
SsaComputeImpl::variableUse(v, use, b, _) and
reachesEndOfBlock(v, defbb, defindex, b.getAPredecessor()) and
not SsaComputeImpl::ssaDefReachesUseWithinBlock(v, b, _, use)
)
}
/***
/**
* Holds if `(defbb, defindex)` is an SSA definition of `v` that reaches an exit without crossing another
* SSA definition of `v`.
*/
@@ -315,9 +347,7 @@ private cached module SsaComputeImpl {
SsaComputeImpl::variableUse(v, use, last, index)
)
}
}
}
import SsaComputeImpl::SsaDefinitionsImpl as SsaDefinitions
@@ -325,9 +355,8 @@ import SsaComputeImpl::EssaDefinitionsImpl as EssaDefinitions
import SsaComputeImpl::LivenessImpl as Liveness
/* This is exported primarily for testing */
/* A note on numbering
/*
* A note on numbering
* In order to create an SSA graph, we need an order of definitions and uses within a basic block.
* To do this we index definitions and uses as follows:
* Phi-functions have an index of -1, so precede all normal uses and definitions in a block.
@@ -337,10 +366,8 @@ import SsaComputeImpl::LivenessImpl as Liveness
* * a definition at the `j`th node of a block is given the index `2*j + 1`.
*/
pragma [inline]
pragma[inline]
int phiIndex() { result = -1 }
pragma [inline]
pragma[inline]
int piIndex() { result = -2 }

72
python/ql/src/semmle/python/essa/SsaDefinitions.qll
View File

@@ -1,44 +1,48 @@
/** Provides classes and predicates for determining the uses and definitions of
/**
* Provides classes and predicates for determining the uses and definitions of
* variables for ESSA form.
*/
import python
private import semmle.python.pointsto.Base
cached module SsaSource {
cached
module SsaSource {
/** Holds if `v` is used as the receiver in a method call. */
cached predicate method_call_refinement(Variable v, ControlFlowNode use, CallNode call) {
cached
predicate method_call_refinement(Variable v, ControlFlowNode use, CallNode call) {
use = v.getAUse() and
call.getFunction().(AttrNode).getObject() = use and
not test_contains(_, call)
}
/** Holds if `v` is defined by assignment at `defn` and given `value`. */
cached predicate assignment_definition(Variable v, ControlFlowNode defn, ControlFlowNode value) {
cached
predicate assignment_definition(Variable v, ControlFlowNode defn, ControlFlowNode value) {
defn.(NameNode).defines(v) and defn.(DefinitionNode).getValue() = value
}
/** Holds if `v` is defined by assignment of the captured exception. */
cached predicate exception_capture(Variable v, NameNode defn) {
cached
predicate exception_capture(Variable v, NameNode defn) {
defn.defines(v) and
exists(ExceptFlowNode ex | ex.getName() = defn)
}
/** Holds if `v` is defined by a with statement. */
cached predicate with_definition(Variable v, ControlFlowNode defn) {
cached
predicate with_definition(Variable v, ControlFlowNode defn) {
exists(With with, Name var |
with.getOptionalVars() = var and
var.getAFlowNode() = defn |
var.getAFlowNode() = defn
|
var = v.getAStore()
)
}
/** Holds if `v` is defined by multiple assignment at `defn`. */
cached predicate multi_assignment_definition(Variable v, ControlFlowNode defn, int n, SequenceNode lhs) {
cached
predicate multi_assignment_definition(Variable v, ControlFlowNode defn, int n, SequenceNode lhs) {
(
defn.(NameNode).defines(v)
or
@@ -50,44 +54,52 @@ cached module SsaSource {
}
/** Holds if `v` is defined by a `for` statement, the definition being `defn` */
cached predicate iteration_defined_variable(Variable v, ControlFlowNode defn, ControlFlowNode sequence) {
cached
predicate iteration_defined_variable(Variable v, ControlFlowNode defn, ControlFlowNode sequence) {
exists(ForNode for | for.iterates(defn, sequence)) and
defn.(NameNode).defines(v)
}
/** Holds if `v` is a parameter variable and `defn` is the CFG node for that parameter. */
cached predicate parameter_definition(Variable v, ControlFlowNode defn) {
cached
predicate parameter_definition(Variable v, ControlFlowNode defn) {
exists(Function f, Name param |
f.getAnArg() = param or
f.getVararg() = param or
f.getKwarg() = param or
f.getKeywordOnlyArg(_) = param |
f.getKeywordOnlyArg(_) = param
|
defn.getNode() = param and
param.getVariable() = v
)
}
/** Holds if `v` is deleted at `del`. */
cached predicate deletion_definition(Variable v, DeletionNode del) {
cached
predicate deletion_definition(Variable v, DeletionNode del) {
del.getTarget().(NameNode).deletes(v)
}
/** Holds if the name of `var` refers to a submodule of a package and `f` is the entry point
/**
* Holds if the name of `var` refers to a submodule of a package and `f` is the entry point
* to the __init__ module of that package.
*/
cached predicate init_module_submodule_defn(SsaSourceVariable var, ControlFlowNode f) {
cached
predicate init_module_submodule_defn(SsaSourceVariable var, ControlFlowNode f) {
var instanceof GlobalVariable and
exists(Module init |
init.isPackageInit() and exists(init.getPackage().getSubModule(var.getName())) and
init.isPackageInit() and
exists(init.getPackage().getSubModule(var.getName())) and
init.getEntryNode() = f and
var.getScope() = init
)
}
/** Holds if the `v` is in scope at a `from import ... *` and may thus be redefined by that statement */
cached predicate import_star_refinement(SsaSourceVariable v, ControlFlowNode use, ControlFlowNode def) {
use = def and def instanceof ImportStarNode
and
cached
predicate import_star_refinement(SsaSourceVariable v, ControlFlowNode use, ControlFlowNode def) {
use = def and
def instanceof ImportStarNode and
(
v.getScope() = def.getScope()
or
@@ -99,13 +111,16 @@ cached module SsaSource {
}
/** Holds if an attribute is assigned at `def` and `use` is the use of `v` for that assignment */
cached predicate attribute_assignment_refinement(Variable v, ControlFlowNode use, ControlFlowNode def) {
cached
predicate attribute_assignment_refinement(Variable v, ControlFlowNode use, ControlFlowNode def) {
use.(NameNode).uses(v) and
def.isStore() and def.(AttrNode).getObject() = use
def.isStore() and
def.(AttrNode).getObject() = use
}
/** Holds if a `v` is used as an argument to `call`, which *may* modify the object referred to by `v` */
cached predicate argument_refinement(Variable v, ControlFlowNode use, CallNode call) {
cached
predicate argument_refinement(Variable v, ControlFlowNode use, CallNode call) {
use.(NameNode).uses(v) and
call.getArg(0) = use and
not method_call_refinement(v, _, call) and
@@ -113,13 +128,15 @@ cached module SsaSource {
}
/** Holds if an attribute is deleted at `def` and `use` is the use of `v` for that deletion */
cached predicate attribute_deletion_refinement(Variable v, NameNode use, DeletionNode def) {
cached
predicate attribute_deletion_refinement(Variable v, NameNode use, DeletionNode def) {
use.uses(v) and
def.getTarget().(AttrNode).getObject() = use
}
/** Holds if the set of possible values for `v` is refined by `test` and `use` is the use of `v` in that test. */
cached predicate test_refinement(Variable v, ControlFlowNode use, ControlFlowNode test) {
cached
predicate test_refinement(Variable v, ControlFlowNode use, ControlFlowNode test) {
use.(NameNode).uses(v) and
test.getAChild*() = use and
test.isBranch() and
@@ -129,5 +146,4 @@ cached module SsaSource {
not block.getLastNode() = test
)
}
}

127
python/ql/src/semmle/python/filters/GeneratedCode.qll
View File

@@ -5,105 +5,118 @@ import semmle.python.templates.Templates
* A file that is detected as being generated.
*/
abstract class GeneratedFile extends File {
abstract string getTool();
}
/* We distinguish between a "lax" match which just includes "generated by" or similar versus a "strict" match which includes "this file is generated by" or similar
/*
* We distinguish between a "lax" match which just includes "generated by" or similar versus a "strict" match which includes "this file is generated by" or similar
* "lax" matches are taken to indicate generated file if they occur at the top of a file. "strict" matches can occur anywhere.
* There is no formal reason for the above, it just seems to work well in practice.
*/
library class GenericGeneratedFile extends GeneratedFile {
GenericGeneratedFile() {
not this instanceof SpecificGeneratedFile
and
not this instanceof SpecificGeneratedFile and
(
(lax_generated_by(this, _) or lax_generated_from(this, _)) and dont_modify(this)
(lax_generated_by(this, _) or lax_generated_from(this, _)) and
dont_modify(this)
or
strict_generated_by(this, _) or strict_generated_from(this, _)
strict_generated_by(this, _)
or
strict_generated_from(this, _)
or
auto_generated(this)
)
}
override string getTool() {
lax_generated_by(this, result) or strict_generated_by(this, result)
}
override string getTool() { lax_generated_by(this, result) or strict_generated_by(this, result) }
}
private string comment_or_docstring(File f, boolean before_code) {
exists(Comment c |
exists(Comment c |
c.getLocation().getFile() = f and
result = c.getText() |
if exists(Stmt s | s.getEnclosingModule().getFile() = f and s.getLocation().getStartLine() < c.getLocation().getStartLine()) then
before_code = false
else
before_code = true
result = c.getText()
|
if
exists(Stmt s |
s.getEnclosingModule().getFile() = f and
s.getLocation().getStartLine() < c.getLocation().getStartLine()
)
then before_code = false
else before_code = true
)
or
exists(Module m | m.getFile() = f |
result = m.getDocString().getText() and
before_code = true
)
}
private predicate lax_generated_by(File f, string tool) {
exists(string comment | comment = comment_or_docstring(f, _) |
tool = comment.regexpCapture("(?is).*\\b(?:(?:auto[ -]?)?generated|created automatically) by (?:the )?([-/\\w.]+[-/\\w]).*", 1)
tool =
comment
.regexpCapture("(?is).*\\b(?:(?:auto[ -]?)?generated|created automatically) by (?:the )?([-/\\w.]+[-/\\w]).*",
1)
)
}
private predicate lax_generated_from(File f, string src) {
exists(string comment | comment = comment_or_docstring(f, _) |
src = comment.regexpCapture("(?is).*\\b((?:auto[ -]?)?generated|created automatically) from ([-/\\w.]+[-/\\w]).*", 1)
)
exists(string comment | comment = comment_or_docstring(f, _) |
src =
comment
.regexpCapture("(?is).*\\b((?:auto[ -]?)?generated|created automatically) from ([-/\\w.]+[-/\\w]).*",
1)
)
}
private predicate strict_generated_by(File f, string tool) {
exists(string comment | comment = comment_or_docstring(f, true) |
tool = comment.regexpCapture("(?is)# *(?:this +)?(?:(?:code|file) +)?(?:is +)?(?:(?:auto(?:matically)?[ -]?)?generated|created automatically) by (?:the )?([-/\\w.]+[-/\\w]).*", 1)
tool =
comment
.regexpCapture("(?is)# *(?:this +)?(?:(?:code|file) +)?(?:is +)?(?:(?:auto(?:matically)?[ -]?)?generated|created automatically) by (?:the )?([-/\\w.]+[-/\\w]).*",
1)
)
}
private predicate strict_generated_from(File f, string src) {
exists(string comment | comment = comment_or_docstring(f, true) |
src = comment.regexpCapture("(?is)# *(?:this +)?(?:(?:code|file) +)?(?:is +)?(?:(?:auto(?:matically)?[ -]?)?generated|created automatically) from ([-/\\w.]+[-/\\w]).*", 1)
src =
comment
.regexpCapture("(?is)# *(?:this +)?(?:(?:code|file) +)?(?:is +)?(?:(?:auto(?:matically)?[ -]?)?generated|created automatically) from ([-/\\w.]+[-/\\w]).*",
1)
)
}
private predicate dont_modify(File f) {
comment_or_docstring(f, _).regexpMatch("(?is).*\\b(Do not|Don't) (edit|modify|make changes)\\b.*")
comment_or_docstring(f, _).regexpMatch("(?is).*\\b(Do not|Don't) (edit|modify|make changes)\\b.*")
}
private predicate auto_generated(File f) {
exists(Comment c |
c.getLocation().getFile() = f and
c.getText().regexpMatch("(?is)# *this +(code|file) +is +(auto(matically)?[ -]?generated|created automatically).*")
c
.getText()
.regexpMatch("(?is)# *this +(code|file) +is +(auto(matically)?[ -]?generated|created automatically).*")
)
}
/**
* A file generated by a template engine
* A file generated by a template engine
*/
abstract library class SpecificGeneratedFile extends GeneratedFile {
/* Currently cover Spitfire, Pyxl and Mako.
/*
* Currently cover Spitfire, Pyxl and Mako.
* Django templates are not compiled to Python.
* Jinja2 templates are compiled direct to bytecode via the ast.
*/
}
}
/** File generated by the spitfire templating engine */
class SpitfireGeneratedFile extends SpecificGeneratedFile {
SpitfireGeneratedFile() {
exists(Module m |
m.getFile() = this and not m instanceof SpitfireTemplate |
exists(Module m | m.getFile() = this and not m instanceof SpitfireTemplate |
exists(ImportMember template_method, ImportExpr spitfire_runtime_template |
spitfire_runtime_template.getName() = "spitfire.runtime.template" and
template_method.getModule() = spitfire_runtime_template and
@@ -112,28 +125,18 @@ class SpitfireGeneratedFile extends SpecificGeneratedFile {
)
}
override string getTool() {
result = "spitfire"
}
override string getTool() { result = "spitfire" }
}
/** File generated by the pyxl templating engine */
class PyxlGeneratedFile extends SpecificGeneratedFile {
PyxlGeneratedFile() { this.getSpecifiedEncoding() = "pyxl" }
PyxlGeneratedFile() {
this.getSpecifiedEncoding() = "pyxl"
}
override string getTool() {
result = "pyxl"
}
override string getTool() { result = "pyxl" }
}
/** File generated by the mako templating engine */
class MakoGeneratedFile extends SpecificGeneratedFile {
MakoGeneratedFile() {
exists(Module m | m.getFile() = this |
from_mako_import(m) = "runtime" and
@@ -147,38 +150,31 @@ class MakoGeneratedFile extends SpecificGeneratedFile {
) and
exists(Assign a, Name n |
a.getScope() = m and a.getATarget() = n and n.getId() = "_magic_number"
)
)
)
}
override string getTool() {
result = "mako"
}
override string getTool() { result = "mako" }
}
string from_mako_import(Module m) {
exists(ImportMember member, ImportExpr mako |
member.getScope() = m and
member.getModule() = mako and
mako.getName() = "mako" |
result = member.getName()
mako.getName() = "mako"
|
result = member.getName()
)
}
/** File generated by Google's protobuf tool. */
class ProtobufGeneratedFile extends SpecificGeneratedFile {
ProtobufGeneratedFile() {
this.getName().regexpMatch(".*_pb2?.py")
and
exists(Module m |
m.getFile() = this |
exists(ImportExpr imp |
imp.getEnclosingModule() = m |
this.getName().regexpMatch(".*_pb2?.py") and
exists(Module m | m.getFile() = this |
exists(ImportExpr imp | imp.getEnclosingModule() = m |
imp.getImportedModuleName() = "google.net.proto2.python.public"
)
and
) and
exists(AssignStmt a, Name n |
a.getEnclosingModule() = m and
a.getATarget() = n and
@@ -187,8 +183,5 @@ class ProtobufGeneratedFile extends SpecificGeneratedFile {
)
}
override string getTool() {
result = "protobuf"
}
override string getTool() { result = "protobuf" }
}

19
python/ql/src/semmle/python/filters/Tests.qll
View File

@@ -1,12 +1,11 @@
import python
abstract class TestScope extends Scope {}
abstract class TestScope extends Scope { }
// don't extend Class directly to avoid ambiguous method warnings
class UnitTestClass extends TestScope {
UnitTestClass() {
exists(ClassObject c |
this = c.getPyClass() |
exists(ClassObject c | this = c.getPyClass() |
c.getASuperType() = theUnitTestPackage().attr(_)
or
c.getASuperType().getName().toLowerCase() = "testcase"
@@ -14,35 +13,27 @@ class UnitTestClass extends TestScope {
}
}
PackageObject theUnitTestPackage() {
result.getName() = "unittest"
}
PackageObject theUnitTestPackage() { result.getName() = "unittest" }
abstract class Test extends TestScope {}
abstract class Test extends TestScope { }
class UnitTestFunction extends Test {
UnitTestFunction() {
this.getScope+() instanceof UnitTestClass
and
this.getScope+() instanceof UnitTestClass and
this.(Function).getName().matches("test%")
}
}
class PyTestFunction extends Test {
PyTestFunction() {
exists(Module pytest | pytest.getName() = "pytest") and
this.(Function).getName().matches("test%")
}
}
class NoseTestFunction extends Test {
NoseTestFunction() {
exists(Module nose | nose.getName() = "nose") and
this.(Function).getName().matches("test%")
}
}

27
python/ql/src/semmle/python/libraries/Zope.qll
View File

@@ -1,12 +1,10 @@
/** Utilities for handling the zope libraries */
import python
private import semmle.python.pointsto.PointsTo
/** A method that to a sub-class of `zope.interface.Interface` */
deprecated class ZopeInterfaceMethod extends PyFunctionObject {
/** Holds if this method belongs to a class that sub-classes `zope.interface.Interface` */
ZopeInterfaceMethod() {
exists(Object interface, ClassObject owner |
@@ -16,22 +14,17 @@ deprecated class ZopeInterfaceMethod extends PyFunctionObject {
)
}
override int minParameters() {
result = super.minParameters() + 1
}
override int minParameters() { result = super.minParameters() + 1 }
override int maxParameters() {
if exists(this.getFunction().getVararg()) then
result = super.maxParameters()
else
result = super.maxParameters() + 1
if exists(this.getFunction().getVararg())
then result = super.maxParameters()
else result = super.maxParameters() + 1
}
}
/** A method that belongs to a sub-class of `zope.interface.Interface` */
class ZopeInterfaceMethodValue extends PythonFunctionValue {
/** Holds if this method belongs to a class that sub-classes `zope.interface.Interface` */
ZopeInterfaceMethodValue() {
exists(Value interface, ClassValue owner |
@@ -44,15 +37,11 @@ class ZopeInterfaceMethodValue extends PythonFunctionValue {
)
}
override int minParameters() {
result = super.minParameters() + 1
}
override int minParameters() { result = super.minParameters() + 1 }
override int maxParameters() {
if exists(this.getScope().getVararg()) then
result = super.maxParameters()
else
result = super.maxParameters() + 1
if exists(this.getScope().getVararg())
then result = super.maxParameters()
else result = super.maxParameters() + 1
}
}

24
python/ql/src/semmle/python/protocols.qll
View File

@@ -1,31 +1,19 @@
import python
/** Retained for backwards compatibility use ClassObject.isIterator() instead. */
predicate is_iterator(ClassObject c) {
c.isIterator()
}
predicate is_iterator(ClassObject c) { c.isIterator() }
/** Retained for backwards compatibility use ClassObject.isIterable() instead. */
predicate is_iterable(ClassObject c) {
c.isIterable()
}
predicate is_iterable(ClassObject c) { c.isIterable() }
/** Retained for backwards compatibility use ClassObject.isCollection() instead. */
predicate is_collection(ClassObject c) {
c.isCollection()
}
predicate is_collection(ClassObject c) { c.isCollection() }
/** Retained for backwards compatibility use ClassObject.isMapping() instead. */
predicate is_mapping(ClassObject c) {
c.isMapping()
}
predicate is_mapping(ClassObject c) { c.isMapping() }
/** Retained for backwards compatibility use ClassObject.isSequence() instead. */
predicate is_sequence(ClassObject c) {
c.isSequence()
}
predicate is_sequence(ClassObject c) { c.isSequence() }
/** Retained for backwards compatibility use ClassObject.isContextManager() instead. */
predicate is_context_manager(ClassObject c) {
c.isContextManager()
}
predicate is_context_manager(ClassObject c) { c.isContextManager() }

416
python/ql/src/semmle/python/regex.qll
View File

@@ -2,13 +2,20 @@ import python
import semmle.python.objects.ObjectInternal
private predicate re_module_function(string name, int flags) {
name = "compile" and flags = 1 or
name = "search" and flags = 2 or
name = "match" and flags = 2 or
name = "split" and flags = 3 or
name = "findall" and flags = 2 or
name = "finditer" and flags = 2 or
name = "sub" and flags = 4 or
name = "compile" and flags = 1
or
name = "search" and flags = 2
or
name = "match" and flags = 2
or
name = "split" and flags = 3
or
name = "findall" and flags = 2
or
name = "finditer" and flags = 2
or
name = "sub" and flags = 4
or
name = "subn" and flags = 4
}
@@ -17,16 +24,15 @@ private predicate re_module_function(string name, int flags) {
* If regex mode is not known, `mode` will be `"None"`.
*/
predicate used_as_regex(Expr s, string mode) {
(s instanceof Bytes or s instanceof Unicode)
and
(s instanceof Bytes or s instanceof Unicode) and
/* Call to re.xxx(regex, ... [mode]) */
exists(CallNode call, string name |
call.getArg(0).refersTo(_, _, s.getAFlowNode()) and
call.getFunction().pointsTo(Module::named("re").attr(name)) |
call.getFunction().pointsTo(Module::named("re").attr(name))
|
mode = "None"
or
exists(Value obj |
mode = mode_from_mode_object(obj) |
exists(Value obj | mode = mode_from_mode_object(obj) |
exists(int flags_arg |
re_module_function(name, flags_arg) and
call.getArg(flags_arg).pointsTo(obj)
@@ -39,30 +45,30 @@ predicate used_as_regex(Expr s, string mode) {
string mode_from_mode_object(Value obj) {
(
result = "DEBUG" or result = "IGNORECASE" or result = "LOCALE" or
result = "MULTILINE" or result = "DOTALL" or result = "UNICODE" or
result = "DEBUG" or
result = "IGNORECASE" or
result = "LOCALE" or
result = "MULTILINE" or
result = "DOTALL" or
result = "UNICODE" or
result = "VERBOSE"
) and
exists(int flag |
flag = Value::named("sre_constants.SRE_FLAG_" + result).(ObjectInternal).intValue()
and
flag = Value::named("sre_constants.SRE_FLAG_" + result).(ObjectInternal).intValue() and
obj.(ObjectInternal).intValue().bitAnd(flag) = flag
)
}
/** A StrConst used as a regular expression */
abstract class RegexString extends Expr {
RegexString() {
(this instanceof Bytes or this instanceof Unicode)
}
RegexString() { (this instanceof Bytes or this instanceof Unicode) }
predicate char_set_start(int start, int end) {
this.nonEscapedCharAt(start) = "[" and
(
this.getChar(start+1) = "^" and end = start + 2
or
not this.getChar(start+1) = "^" and end = start + 1
this.getChar(start + 1) = "^" and end = start + 2
or
not this.getChar(start + 1) = "^" and end = start + 1
)
}
@@ -70,58 +76,47 @@ abstract class RegexString extends Expr {
predicate charSet(int start, int end) {
exists(int inner_start, int inner_end |
this.char_set_start(start, inner_start) and
not this.char_set_start(_, start) |
end = inner_end + 1 and inner_end > inner_start and
not this.char_set_start(_, start)
|
end = inner_end + 1 and
inner_end > inner_start and
this.nonEscapedCharAt(inner_end) = "]" and
not exists(int mid | this.nonEscapedCharAt(mid) = "]" |
mid > inner_start and mid < inner_end
)
not exists(int mid | this.nonEscapedCharAt(mid) = "]" | mid > inner_start and mid < inner_end)
)
}
predicate escapingChar(int pos) {
this.escaping(pos) = true
}
predicate escapingChar(int pos) { this.escaping(pos) = true }
private boolean escaping(int pos) {
pos = -1 and result = false
or
this.getChar(pos) = "\\" and result = this.escaping(pos-1).booleanNot()
this.getChar(pos) = "\\" and result = this.escaping(pos - 1).booleanNot()
or
this.getChar(pos) != "\\" and result = false
}
/** Gets the text of this regex */
string getText() {
result = ((Unicode)this).getS()
result = this.(Unicode).getS()
or
result = ((Bytes)this).getS()
result = this.(Bytes).getS()
}
string getChar(int i) {
result = this.getText().charAt(i)
}
string getChar(int i) { result = this.getText().charAt(i) }
string nonEscapedCharAt(int i) {
result = this.getText().charAt(i) and
not this.escapingChar(i-1)
not this.escapingChar(i - 1)
}
private predicate isOptionDivider(int i) {
this.nonEscapedCharAt(i) = "|"
}
private predicate isOptionDivider(int i) { this.nonEscapedCharAt(i) = "|" }
private predicate isGroupEnd(int i) {
this.nonEscapedCharAt(i) = ")"
}
private predicate isGroupEnd(int i) { this.nonEscapedCharAt(i) = ")" }
private predicate isGroupStart(int i) {
this.nonEscapedCharAt(i) = "("
}
private predicate isGroupStart(int i) { this.nonEscapedCharAt(i) = "(" }
predicate failedToParse(int i) {
exists(this.getChar(i))
and
exists(this.getChar(i)) and
not exists(int start, int end |
this.top_level(start, end) and
start <= i and
@@ -130,42 +125,47 @@ abstract class RegexString extends Expr {
}
private predicate escapedCharacter(int start, int end) {
this.escapingChar(start) and not exists(this.getText().substring(start+1, end+1).toInt()) and
this.escapingChar(start) and
not exists(this.getText().substring(start + 1, end + 1).toInt()) and
(
this.getChar(start+1) = "x" and end = start + 4
this.getChar(start + 1) = "x" and end = start + 4
or
end in [start+2..start+4] and
exists(this.getText().substring(start+1, end).toInt())
end in [start + 2 .. start + 4] and
exists(this.getText().substring(start + 1, end).toInt())
or
this.getChar(start+1) != "x" and end = start + 2
this.getChar(start + 1) != "x" and end = start + 2
)
}
private predicate inCharSet(int index) {
exists(int x, int y | this.charSet(x, y) and index in [x+1 .. y-2])
exists(int x, int y | this.charSet(x, y) and index in [x + 1 .. y - 2])
}
/* 'simple' characters are any that don't alter the parsing of the regex.
/*
* 'simple' characters are any that don't alter the parsing of the regex.
*/
private predicate simpleCharacter(int start, int end) {
end = start+1 and
end = start + 1 and
not this.charSet(start, _) and
not this.charSet(_, start+1) and
exists(string c |
c = this.getChar(start) |
not this.charSet(_, start + 1) and
exists(string c | c = this.getChar(start) |
exists(int x, int y, int z |
this.charSet(x, z) and
this.char_set_start(x, y) |
this.charSet(x, z) and
this.char_set_start(x, y)
|
start = y
or
start = z-2
start = z - 2
or
start > y and start < z-2 and not c = "-"
start > y and start < z - 2 and not c = "-"
)
or
not this.inCharSet(start) and
not c = "(" and not c = "[" and
not c = ")" and not c = "|" and
not c = "(" and
not c = "[" and
not c = ")" and
not c = "|" and
not this.qualifier(start, _, _)
)
}
@@ -176,28 +176,20 @@ abstract class RegexString extends Expr {
not exists(int x, int y | this.escapedCharacter(x, y) and x <= start and y >= end)
or
this.escapedCharacter(start, end)
)
and
not exists(int x, int y |
this.group_start(x, y) and x <= start and y >= end
)
) and
not exists(int x, int y | this.group_start(x, y) and x <= start and y >= end)
}
predicate normalCharacter(int start, int end) {
this.character(start, end)
and
this.character(start, end) and
not this.specialCharacter(start, end, _)
}
predicate specialCharacter(int start, int end, string char) {
this.character(start, end)
and
end = start+1
and
char = this.getChar(start)
and
(char = "$" or char = "^" or char = ".")
and
this.character(start, end) and
end = start + 1 and
char = this.getChar(start) and
(char = "$" or char = "^" or char = ".") and
not this.inCharSet(start)
}
@@ -210,17 +202,17 @@ abstract class RegexString extends Expr {
/** Gets the number of the group in start,end */
int getGroupNumber(int start, int end) {
this.group(start, end) and
result = count(int i | this.group(i, _) and i < start and not this.non_capturing_group_start(i, _)) + 1
this.group(start, end) and
result =
count(int i | this.group(i, _) and i < start and not this.non_capturing_group_start(i, _)) + 1
}
/** Gets the name, if it has one, of the group in start,end */
string getGroupName(int start, int end) {
this.group(start, end)
and
this.group(start, end) and
exists(int name_end |
this.named_group_start(start, name_end) and
result = this.getText().substring(start+4, name_end-1)
result = this.getText().substring(start + 4, name_end - 1)
)
}
@@ -236,8 +228,7 @@ abstract class RegexString extends Expr {
}
private predicate emptyGroup(int start, int end) {
exists(int endm1 |
end = endm1+1 |
exists(int endm1 | end = endm1 + 1 |
this.group_start(start, endm1) and
this.isGroupEnd(endm1)
)
@@ -263,21 +254,20 @@ abstract class RegexString extends Expr {
exists(int in_start |
this.negative_lookahead_assertion_start(start, in_start)
or
this.negative_lookbehind_assertion_start(start, in_start) |
this.negative_lookbehind_assertion_start(start, in_start)
|
this.groupContents(start, end, in_start, _)
)
}
private predicate positiveLookaheadAssertionGroup(int start, int end) {
exists(int in_start |
this.lookahead_assertion_start(start, in_start) |
exists(int in_start | this.lookahead_assertion_start(start, in_start) |
this.groupContents(start, end, in_start, _)
)
}
private predicate positiveLookbehindAssertionGroup(int start, int end) {
exists(int in_start |
this.lookbehind_assertion_start(start, in_start) |
exists(int in_start | this.lookbehind_assertion_start(start, in_start) |
this.groupContents(start, end, in_start, _)
)
}
@@ -306,42 +296,43 @@ abstract class RegexString extends Expr {
private predicate non_capturing_group_start(int start, int end) {
this.isGroupStart(start) and
this.getChar(start+1) = "?" and
this.getChar(start+2) = ":" and
end = start+3
this.getChar(start + 1) = "?" and
this.getChar(start + 2) = ":" and
end = start + 3
}
private predicate simple_group_start(int start, int end) {
this.isGroupStart(start) and
this.getChar(start+1) != "?" and end = start+1
this.getChar(start + 1) != "?" and
end = start + 1
}
private predicate named_group_start(int start, int end) {
this.isGroupStart(start) and
this.getChar(start+1) = "?" and
this.getChar(start+2) = "P" and
this.getChar(start+3) = "<" and
not this.getChar(start+4) = "=" and
not this.getChar(start+4) = "!" and
this.getChar(start + 1) = "?" and
this.getChar(start + 2) = "P" and
this.getChar(start + 3) = "<" and
not this.getChar(start + 4) = "=" and
not this.getChar(start + 4) = "!" and
exists(int name_end |
name_end = min(int i | i > start+4 and this.getChar(i) = ">") and
name_end = min(int i | i > start + 4 and this.getChar(i) = ">") and
end = name_end + 1
)
}
private predicate named_backreference_start(int start, int end) {
this.isGroupStart(start) and
this.getChar(start+1) = "?" and
this.getChar(start+2) = "P" and
this.getChar(start+3) = "=" and
end = min(int i | i > start+4 and this.getChar(i) = "?")
this.getChar(start + 1) = "?" and
this.getChar(start + 2) = "P" and
this.getChar(start + 3) = "=" and
end = min(int i | i > start + 4 and this.getChar(i) = "?")
}
private predicate flag_group_start(int start, int end, string c) {
this.isGroupStart(start) and
this.getChar(start+1) = "?" and
end = start+3 and
c = this.getChar(start+2) and
this.getChar(start + 1) = "?" and
end = start + 3 and
c = this.getChar(start + 2) and
(
c = "i" or
c = "L" or
@@ -352,12 +343,12 @@ abstract class RegexString extends Expr {
)
}
/** Gets the mode of this regular expression string if
/**
* Gets the mode of this regular expression string if
* it is defined by a prefix.
*/
string getModeFromPrefix() {
exists(string c |
this.flag_group_start(_, _, c) |
exists(string c | this.flag_group_start(_, _, c) |
c = "i" and result = "IGNORECASE"
or
c = "L" and result = "LOCALE"
@@ -366,7 +357,7 @@ abstract class RegexString extends Expr {
or
c = "s" and result = "DOTALL"
or
c = "u" and result = "UNICODE"
c = "u" and result = "UNICODE"
or
c = "x" and result = "VERBOSE"
)
@@ -374,39 +365,39 @@ abstract class RegexString extends Expr {
private predicate lookahead_assertion_start(int start, int end) {
this.isGroupStart(start) and
this.getChar(start+1) = "?" and
this.getChar(start+2) = "=" and
end = start+3
this.getChar(start + 1) = "?" and
this.getChar(start + 2) = "=" and
end = start + 3
}
private predicate negative_lookahead_assertion_start(int start, int end) {
this.isGroupStart(start) and
this.getChar(start+1) = "?" and
this.getChar(start+2) = "!" and
end = start+3
this.getChar(start + 1) = "?" and
this.getChar(start + 2) = "!" and
end = start + 3
}
private predicate lookbehind_assertion_start(int start, int end) {
this.isGroupStart(start) and
this.getChar(start+1) = "?" and
this.getChar(start+2) = "<" and
this.getChar(start+3) = "=" and
end = start+4
this.getChar(start + 1) = "?" and
this.getChar(start + 2) = "<" and
this.getChar(start + 3) = "=" and
end = start + 4
}
private predicate negative_lookbehind_assertion_start(int start, int end) {
this.isGroupStart(start) and
this.getChar(start+1) = "?" and
this.getChar(start+2) = "<" and
this.getChar(start+3) = "!" and
end = start+4
this.getChar(start + 1) = "?" and
this.getChar(start + 2) = "<" and
this.getChar(start + 3) = "!" and
end = start + 4
}
private predicate comment_group_start(int start, int end) {
this.isGroupStart(start) and
this.getChar(start+1) = "?" and
this.getChar(start+2) = "#" and
end = start+3
this.getChar(start + 1) = "?" and
this.getChar(start + 2) = "#" and
end = start + 3
}
predicate groupContents(int start, int end, int in_start, int in_end) {
@@ -417,20 +408,21 @@ abstract class RegexString extends Expr {
}
private predicate named_backreference(int start, int end, string name) {
this.named_backreference_start(start, start+4) and
end = min(int i | i > start+4 and this.getChar(i) = ")") + 1 and
name = this.getText().substring(start+4, end-2)
this.named_backreference_start(start, start + 4) and
end = min(int i | i > start + 4 and this.getChar(i) = ")") + 1 and
name = this.getText().substring(start + 4, end - 2)
}
private predicate numbered_backreference(int start, int end, int value) {
this.escapingChar(start)
and
this.escapingChar(start) and
exists(string text, string svalue, int len |
end = start + len and
text = this.getText() and len in [2..3] |
svalue = text.substring(start+1, start+len) and
text = this.getText() and
len in [2 .. 3]
|
svalue = text.substring(start + 1, start + len) and
value = svalue.toInt() and
not exists(text.substring(start+1, start+len+1).toInt()) and
not exists(text.substring(start + 1, start + len + 1).toInt()) and
value != 0
)
}
@@ -443,17 +435,14 @@ abstract class RegexString extends Expr {
}
/** Gets the number of the back reference in start,end */
int getBackrefNumber(int start, int end) {
this.numbered_backreference(start, end, result)
}
int getBackrefNumber(int start, int end) { this.numbered_backreference(start, end, result) }
/** Gets the name, if it has one, of the back reference in start,end */
string getBackrefName(int start, int end) {
this.named_backreference(start, end, result)
}
string getBackrefName(int start, int end) { this.named_backreference(start, end, result) }
private predicate baseItem(int start, int end) {
this.character(start, end) and not exists(int x, int y | this.charSet(x, y) and x <= start and y >= end)
this.character(start, end) and
not exists(int x, int y | this.charSet(x, y) and x <= start and y >= end)
or
this.group(start, end)
or
@@ -463,12 +452,8 @@ abstract class RegexString extends Expr {
private predicate qualifier(int start, int end, boolean maybe_empty) {
this.short_qualifier(start, end, maybe_empty) and not this.getChar(end) = "?"
or
exists(int short_end |
this.short_qualifier(start, short_end, maybe_empty) |
if this.getChar(short_end) = "?" then
end = short_end+1
else
end = short_end
exists(int short_end | this.short_qualifier(start, short_end, maybe_empty) |
if this.getChar(short_end) = "?" then end = short_end + 1 else end = short_end
)
}
@@ -479,26 +464,28 @@ abstract class RegexString extends Expr {
this.getChar(start) = "*" and maybe_empty = true
or
this.getChar(start) = "?" and maybe_empty = true
) and end = start + 1
) and
end = start + 1
or
exists(int endin | end = endin + 1 |
this.getChar(start) = "{" and this.getChar(endin) = "}" and
this.getChar(start) = "{" and
this.getChar(endin) = "}" and
end > start and
exists(string multiples |
multiples = this.getText().substring(start+1, endin) |
exists(string multiples | multiples = this.getText().substring(start + 1, endin) |
multiples.regexpMatch("0*,[0-9]*") and maybe_empty = true
or
multiples.regexpMatch("0*[1-9][0-9]*,[0-9]*") and maybe_empty = false
)
and
) and
not exists(int mid |
this.getChar(mid) = "}" and
mid > start and mid < endin
mid > start and
mid < endin
)
)
}
/** Whether the text in the range start,end is a qualified item, where item is a character,
/**
* Whether the text in the range start,end is a qualified item, where item is a character,
* a character set or a group.
*/
predicate qualifiedItem(int start, int end, boolean maybe_empty) {
@@ -520,19 +507,18 @@ abstract class RegexString extends Expr {
(
start = 0 or
this.group_start(_, start) or
this.isOptionDivider(start-1)
)
and
this.item(start, end) or
(
exists(int mid |
this.subsequence(start, mid) and
this.item(mid, end)
)
this.isOptionDivider(start - 1)
) and
this.item(start, end)
or
exists(int mid |
this.subsequence(start, mid) and
this.item(mid, end)
)
}
/** Whether the text in the range start,end is a sequence of 1 or more items, where an item is a character,
/**
* Whether the text in the range start,end is a sequence of 1 or more items, where an item is a character,
* a character set or a group.
*/
predicate sequence(int start, int end) {
@@ -552,9 +538,12 @@ abstract class RegexString extends Expr {
}
private predicate item_end(int end) {
this.character(_, end) or
exists(int endm1 | this.isGroupEnd(endm1) and end = endm1 + 1) or
this.charSet(_, end) or
this.character(_, end)
or
exists(int endm1 | this.isGroupEnd(endm1) and end = endm1 + 1)
or
this.charSet(_, end)
or
this.qualifier(_, end, _)
}
@@ -564,30 +553,36 @@ abstract class RegexString extends Expr {
}
private predicate subalternation(int start, int end, int item_start) {
this.sequenceOrQualified(start, end) and not this.isOptionDivider(start-1) and
this.sequenceOrQualified(start, end) and
not this.isOptionDivider(start - 1) and
item_start = start
or
start = end and not this.item_end(start) and this.isOptionDivider(end) and
start = end and
not this.item_end(start) and
this.isOptionDivider(end) and
item_start = start
or
exists(int mid |
this.subalternation(start, mid, _) and
this.isOptionDivider(mid) and
item_start = mid+1 |
item_start = mid + 1
|
this.sequenceOrQualified(item_start, end)
or
not this.item_start(end) and end = item_start
)
}
/** Whether the text in the range start,end is an alternation
/**
* Whether the text in the range start,end is an alternation
*/
predicate alternation(int start, int end) {
this.top_level(start, end) and
exists(int less | this.subalternation(start, less, _) and less < end)
}
/** Whether the text in the range start,end is an alternation and the text in part_start, part_end is one of the
/**
* Whether the text in the range start,end is an alternation and the text in part_start, part_end is one of the
* options in that alternation.
*/
predicate alternationOption(int start, int end, int part_start, int part_end) {
@@ -599,22 +594,19 @@ abstract class RegexString extends Expr {
private predicate firstPart(int start, int end) {
start = 0 and end = this.getText().length()
or
exists(int x |
this.firstPart(x, end) |
exists(int x | this.firstPart(x, end) |
this.emptyMatchAtStartGroup(x, start) or
this.qualifiedItem(x, start, true) or
this.specialCharacter(x, start, "^")
)
or
exists(int y |
this.firstPart(start, y) |
exists(int y | this.firstPart(start, y) |
this.item(start, end)
or
this.qualifiedPart(start, end, y, _)
)
or
exists(int x, int y |
this.firstPart(x, y) |
exists(int x, int y | this.firstPart(x, y) |
this.groupContents(x, y, start, end)
or
this.alternationOption(x, y, start, end)
@@ -625,8 +617,7 @@ abstract class RegexString extends Expr {
private predicate lastPart(int start, int end) {
start = 0 and end = this.getText().length()
or
exists(int y |
this.lastPart(start, y) |
exists(int y | this.lastPart(start, y) |
this.emptyMatchAtEndGroup(end, y) or
this.qualifiedItem(end, y, true) or
this.specialCharacter(end, y, "$")
@@ -637,60 +628,52 @@ abstract class RegexString extends Expr {
this.item(start, end)
)
or
exists(int y |
this.lastPart(start, y) |
this.qualifiedPart(start, end, y, _)
)
exists(int y | this.lastPart(start, y) | this.qualifiedPart(start, end, y, _))
or
exists(int x, int y |
this.lastPart(x, y) |
exists(int x, int y | this.lastPart(x, y) |
this.groupContents(x, y, start, end)
or
this.alternationOption(x, y, start, end)
)
}
/** Whether the item at [start, end) is one of the first items
/**
* Whether the item at [start, end) is one of the first items
* to be matched.
*/
predicate firstItem(int start, int end) {
(
this.character(start, end)
or
this.qualifiedItem(start, end, _)
or
this.charSet(start, end)
)
and
this.character(start, end)
or
this.qualifiedItem(start, end, _)
or
this.charSet(start, end)
) and
this.firstPart(start, end)
}
/** Whether the item at [start, end) is one of the last items
/**
* Whether the item at [start, end) is one of the last items
* to be matched.
*/
predicate lastItem(int start, int end) {
(
this.character(start, end)
or
this.qualifiedItem(start, end, _)
or
this.charSet(start, end)
)
and
this.character(start, end)
or
this.qualifiedItem(start, end, _)
or
this.charSet(start, end)
) and
this.lastPart(start, end)
}
}
/** A StrConst used as a regular expression */
class Regex extends RegexString {
Regex() { used_as_regex(this, _) }
Regex() {
used_as_regex(this, _)
}
/** Gets a mode (if any) of this regular expression. Can be any of:
/**
* Gets a mode (if any) of this regular expression. Can be any of:
* DEBUG
* IGNORECASE
* LOCALE
@@ -705,5 +688,4 @@ class Regex extends RegexString {
or
result = this.getModeFromPrefix()
}
}
}

42
python/ql/src/semmle/python/strings.qll
View File

@@ -9,23 +9,22 @@ predicate mapping_format(StrConst e) {
}
/*
MAPPING_KEY = "(\\([^)]+\\))?"
CONVERSION_FLAGS = "[#0\\- +]?"
MINIMUM_FIELD_WIDTH = "(\\*|[0-9]*)"
PRECISION = "(\\.(\\*|[0-9]*))?"
LENGTH_MODIFIER = "[hlL]?"
TYPE = "[bdiouxXeEfFgGcrs%]"
*/
* MAPPING_KEY = "(\\([^)]+\\))?"
* CONVERSION_FLAGS = "[#0\\- +]?"
* MINIMUM_FIELD_WIDTH = "(\\*|[0-9]*)"
* PRECISION = "(\\.(\\*|[0-9]*))?"
* LENGTH_MODIFIER = "[hlL]?"
* TYPE = "[bdiouxXeEfFgGcrs%]"
*/
private
string conversion_specifier_string(StrConst e, int number, int position) {
private string conversion_specifier_string(StrConst e, int number, int position) {
exists(string s, string REGEX | s = e.getText() |
REGEX = "%(\\([^)]*\\))?[#0\\- +]*(\\*|[0-9]*)(\\.(\\*|[0-9]*))?(h|H|l|L)?[badiouxXeEfFgGcrs%]" and
result = s.regexpFind(REGEX, number, position))
result = s.regexpFind(REGEX, number, position)
)
}
private
string conversion_specifier(StrConst e, int number) {
private string conversion_specifier(StrConst e, int number) {
result = conversion_specifier_string(e, number, _) and result != "%%"
}
@@ -39,22 +38,17 @@ int illegal_conversion_specifier(StrConst e) {
/** Gets the number of format items in a format string */
int format_items(StrConst e) {
result = count(int i | | conversion_specifier(e, i)) +
// a conversion specifier uses an extra item for each *
count(int i, int j | conversion_specifier(e, i).charAt(j) = "*")
result =
count(int i | | conversion_specifier(e, i)) +
// a conversion specifier uses an extra item for each *
count(int i, int j | conversion_specifier(e, i).charAt(j) = "*")
}
private string str(Expr e) {
result = ((Num)e).getN()
result = e.(Num).getN()
or
result = "'" + ((StrConst)e).getText() + "'"
result = "'" + e.(StrConst).getText() + "'"
}
/** Gets a string representation of an expression more suited for embedding in message strings than .toString() */
string repr(Expr e) {
if exists(str(e)) then
result = str(e)
else
result = e.toString()
}
string repr(Expr e) { if exists(str(e)) then result = str(e) else result = e.toString() }

70
python/ql/src/semmle/python/templates/PyxlTags.qll
View File

@@ -1,32 +1,22 @@
import python
/** A Tag in Pyxl (which gets converted to a call in Python).
*
/**
* A Tag in Pyxl (which gets converted to a call in Python).
*/
class PyxlTag extends Call {
PyxlTag() { pyxl_tag(this, _) }
PyxlTag() {
pyxl_tag(this, _)
}
string getPyxlTagName() {
pyxl_tag(this, result)
}
string getPyxlTagName() { pyxl_tag(this, result) }
/** Gets the pyxl or Python node that is enclosed by this one in the pyxl source */
Expr getEnclosedNode() {
none()
}
Expr getEnclosedNode() { none() }
/** Gets the Python code (if any) that is contained in this pyxl node */
Expr getEnclosedPythonCode() {
result = this.getEnclosedNode() and not result instanceof PyxlTag
or
result = ((PyxlTag)this.getEnclosedNode()).getEnclosedPythonCode()
result = this.getEnclosedNode().(PyxlTag).getEnclosedPythonCode()
}
}
private predicate pyxl_tag(Call c, string name) {
@@ -39,53 +29,33 @@ private predicate pyxl_tag(Call c, string name) {
}
class PyxlHtmlTag extends PyxlTag {
PyxlHtmlTag() { this.getPyxlTagName().prefix(2) = "x_" }
PyxlHtmlTag() {
this.getPyxlTagName().prefix(2) = "x_"
}
string getTagName() {
result = this.getPyxlTagName().suffix(2)
}
string getTagName() { result = this.getPyxlTagName().suffix(2) }
/** Html tags get transformed into a call. This node is the callee function and the enclosed node is an argument. */
override Expr getEnclosedNode() {
exists(Call c |
exists(Call c |
c.getFunc() = this and
result = c.getAnArg()
)
}
}
class PyxlIfTag extends PyxlTag {
PyxlIfTag() { this.getPyxlTagName() = "_push_condition" }
PyxlIfTag() {
this.getPyxlTagName() = "_push_condition"
}
override Expr getEnclosedNode() {
result = this.getAnArg()
}
override Expr getEnclosedNode() { result = this.getAnArg() }
}
class PyxlEndIfTag extends PyxlTag {
PyxlEndIfTag() { this.getPyxlTagName() = "_leave_if" }
PyxlEndIfTag() {
this.getPyxlTagName() = "_leave_if"
}
override Expr getEnclosedNode() {
result = this.getAnArg()
}
override Expr getEnclosedNode() { result = this.getAnArg() }
}
class PyxlRawHtml extends PyxlTag{
PyxlRawHtml() {
this.getPyxlTagName() = "rawhtml"
}
class PyxlRawHtml extends PyxlTag {
PyxlRawHtml() { this.getPyxlTagName() = "rawhtml" }
/** The text for this raw html, if it is simple text. */
string getText() {
@@ -95,13 +65,7 @@ class PyxlRawHtml extends PyxlTag{
)
}
Expr getValue() {
result = this.getArg(0)
}
override Expr getEnclosedNode() {
result = this.getAnArg()
}
Expr getValue() { result = this.getArg(0) }
override Expr getEnclosedNode() { result = this.getAnArg() }
}

17
python/ql/src/semmle/python/templates/Templates.qll
View File

@@ -1,24 +1,15 @@
import python
abstract class Template extends Module {
}
abstract class Template extends Module { }
class SpitfireTemplate extends Template {
SpitfireTemplate() {
this.getKind() = "Spitfire template"
}
SpitfireTemplate() { this.getKind() = "Spitfire template" }
}
class PyxlModule extends Template {
PyxlModule() {
PyxlModule() {
exists(Comment c | c.getLocation().getFile() = this.getFile() |
c.getText().regexpMatch("# *coding.*pyxl.*")
)
}
}
}

40
python/ql/src/semmle/python/values/StringAttributes.qll
View File

@@ -1,10 +1,12 @@
import python
predicate string_attribute_all(ControlFlowNode n, string attr) {
(n.getNode() instanceof Unicode or n.getNode() instanceof Bytes) and attr = "const"
(n.getNode() instanceof Unicode or n.getNode() instanceof Bytes) and
attr = "const"
or
exists(Object s |
n.refersTo(s, theBytesType(), _) and attr = "bytes" and
n.refersTo(s, theBytesType(), _) and
attr = "bytes" and
// We are only interested in bytes if they may cause an exception if
// implicitly converted to unicode. ASCII is safe.
not s.(StringObject).isAscii()
@@ -12,23 +14,19 @@ predicate string_attribute_all(ControlFlowNode n, string attr) {
}
predicate tracked_object(ControlFlowNode obj, string attr) {
tracked_object_all(obj, attr)
or
tracked_object_any(obj, attr)
tracked_object_all(obj, attr)
or
tracked_object_any(obj, attr)
}
predicate open_file(Object obj) {
obj.(CallNode).getFunction().refersTo(Object::builtin("open"))
}
predicate open_file(Object obj) { obj.(CallNode).getFunction().refersTo(Object::builtin("open")) }
predicate string_attribute_any(ControlFlowNode n, string attr) {
attr = "user-input" and
exists(Object input |
n.(CallNode).getFunction().refersTo(input) |
if major_version() = 2 then
input = Object::builtin("raw_input")
else
input = Object::builtin("input")
exists(Object input | n.(CallNode).getFunction().refersTo(input) |
if major_version() = 2
then input = Object::builtin("raw_input")
else input = Object::builtin("input")
)
or
attr = "file-input" and
@@ -42,19 +40,13 @@ predicate string_attribute_any(ControlFlowNode n, string attr) {
predicate tracked_object_any(ControlFlowNode obj, string attr) {
string_attribute_any(obj, attr)
or
exists(ControlFlowNode other |
tracking_step(other, obj) |
tracked_object_any(other, attr)
)
exists(ControlFlowNode other | tracking_step(other, obj) | tracked_object_any(other, attr))
}
predicate tracked_object_all(ControlFlowNode obj, string attr) {
string_attribute_all(obj, attr)
or
forex(ControlFlowNode other |
tracking_step(other, obj) |
tracked_object_all(other, attr)
)
forex(ControlFlowNode other | tracking_step(other, obj) | tracked_object_all(other, attr))
}
predicate tracked_call_step(ControlFlowNode ret, ControlFlowNode call) {
@@ -72,7 +64,7 @@ ControlFlowNode sequence_for_iterator(ControlFlowNode f) {
)
}
pragma [noinline]
pragma[noinline]
private predicate tracking_step(ControlFlowNode src, ControlFlowNode dest) {
src = dest.(BinaryExprNode).getAnOperand()
or
@@ -86,5 +78,5 @@ private predicate tracking_step(ControlFlowNode src, ControlFlowNode dest) {
or
tracked_call_step(src, dest)
or
dest.refersTo((Object)src)
dest.refersTo(src.(Object))
}