hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2026-04-29 02:35:15 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into jorgectf/python/insecure-cookie

Browse Source
This commit is contained in:
jorgectf
2021-10-27 19:00:55 +02:00
parent 54ed25a925 c9b50f3c2f
commit 28ec8c9dee
5978 changed files with 580332 additions and 251971 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
python/ql/src/Classes/UselessClass.ql
View File

@@ -52,11 +52,7 @@ predicate is_stateful(Class c) {
call.getFunc() = a and
a.getName() = name
|
name = "pop" or
name = "remove" or
name = "discard" or
name = "extend" or
name = "append"
name in ["pop", "remove", "discard", "extend", "append"]
)
}

20
python/ql/src/Customizations.qll
View File

@@ -1,20 +0,0 @@
/**
* Contains customizations to the standard library.
*
* This module is imported by `python.qll`, so any customizations defined here automatically
* apply to all queries.
*
* Typical examples of customizations include adding new subclasses of abstract classes such as
* the `RemoteFlowSource::Range` and `AdditionalTaintStep` classes associated with the security
* queries to model frameworks that are not covered by the standard library.
*/
import python
/* General import that is useful */
// import semmle.python.dataflow.new.DataFlow
//
/* for extending `TaintTracking::AdditionalTaintStep` */
// import semmle.python.dataflow.new.TaintTracking
//
/* for extending `RemoteFlowSource::Range` */
// import semmle.python.dataflow.new.RemoteFlowSources

23
python/ql/src/Diagnostics/ExtractionErrors.ql
View File

@@ -1,23 +0,0 @@
/**
* @name Python extraction errors
* @description List all extraction errors for Python files in the source code directory.
* @kind diagnostic
* @id py/diagnostics/extraction-errors
*/
import python
/**
* Gets the SARIF severity for errors.
*
* See point 3.27.10 in https://docs.oasis-open.org/sarif/sarif/v2.0/sarif-v2.0.html for
* what error means.
*/
int getErrorSeverity() { result = 2 }
from SyntaxError error, File file
where
file = error.getFile() and
exists(file.getRelativePath())
select error, "Extraction failed in " + file + " with error " + error.getMessage(),
getErrorSeverity()

36
python/ql/src/Diagnostics/ExtractionWarnings.ql Normal file
View File

@@ -0,0 +1,36 @@
/**
* @name Python extraction warnings
* @description List all extraction warnings for Python files in the source code directory.
* @kind diagnostic
* @id py/diagnostics/extraction-warnings
*/
import python
/**
* Gets the SARIF severity for warnings.
*
* See https://docs.oasis-open.org/sarif/sarif/v2.1.0/csprd01/sarif-v2.1.0-csprd01.html#_Toc10541338
*/
int getWarningSeverity() { result = 1 }
// The spec
// https://docs.oasis-open.org/sarif/sarif/v2.1.0/csprd01/sarif-v2.1.0-csprd01.html#_Toc10541338
// defines error and warning as:
//
// "error": A serious problem was found. The condition encountered by the tool resulted
// in the analysis being halted or caused the results to be incorrect or incomplete.
//
// "warning": A problem that is not considered serious was found. The condition
// encountered by the tool is such that it is uncertain whether a problem occurred, or
// is such that the analysis might be incomplete but the results that were generated are
// probably valid.
//
// So SyntaxErrors are reported at the warning level, since analysis might be incomplete
// but the results that were generated are probably valid.
from SyntaxError error, File file
where
file = error.getFile() and
exists(file.getRelativePath())
select error, "Extraction failed in " + file + " with error " + error.getMessage(),
getWarningSeverity()

51
python/ql/src/Functions/IncorrectRaiseInSpecialMethod.ql
View File

@@ -22,49 +22,14 @@ private predicate indexing_method(string name) {
}
private predicate arithmetic_method(string name) {
name = "__add__" or
name = "__sub__" or
name = "__div__" or
name = "__pos__" or
name = "__abs__" or
name = "__floordiv__" or
name = "__div__" or
name = "__divmod__" or
name = "__lshift__" or
name = "__and__" or
name = "__or__" or
name = "__xor__" or
name = "__rshift__" or
name = "__pow__" or
name = "__mul__" or
name = "__neg__" or
name = "__radd__" or
name = "__rsub__" or
name = "__rdiv__" or
name = "__rfloordiv__" or
name = "__rdiv__" or
name = "__rlshift__" or
name = "__rand__" or
name = "__ror__" or
name = "__rxor__" or
name = "__rrshift__" or
name = "__rpow__" or
name = "__rmul__" or
name = "__truediv__" or
name = "__rtruediv__" or
name = "__iadd__" or
name = "__isub__" or
name = "__idiv__" or
name = "__ifloordiv__" or
name = "__idiv__" or
name = "__ilshift__" or
name = "__iand__" or
name = "__ior__" or
name = "__ixor__" or
name = "__irshift__" or
name = "__ipow__" or
name = "__imul__" or
name = "__itruediv__"
name in [
"__add__", "__sub__", "__or__", "__xor__", "__rshift__", "__pow__", "__mul__", "__neg__",
"__radd__", "__rsub__", "__rdiv__", "__rfloordiv__", "__div__", "__rdiv__", "__rlshift__",
"__rand__", "__ror__", "__rxor__", "__rrshift__", "__rpow__", "__rmul__", "__truediv__",
"__rtruediv__", "__pos__", "__iadd__", "__isub__", "__idiv__", "__ifloordiv__", "__idiv__",
"__ilshift__", "__iand__", "__ior__", "__ixor__", "__irshift__", "__abs__", "__ipow__",
"__imul__", "__itruediv__", "__floordiv__", "__div__", "__divmod__", "__lshift__", "__and__"
]
}
private predicate ordering_method(string name) {

90
python/ql/src/Functions/ModificationOfParameterWithDefault.ql
View File

@@ -12,88 +12,12 @@
*/
import python
import semmle.python.security.Paths
import semmle.python.functions.ModificationOfParameterWithDefault
import DataFlow::PathGraph
predicate safe_method(string name) {
name = "count" or
name = "index" or
name = "copy" or
name = "get" or
name = "has_key" or
name = "items" or
name = "keys" or
name = "values" or
name = "iteritems" or
name = "iterkeys" or
name = "itervalues" or
name = "__contains__" or
name = "__getitem__" or
name = "__getattribute__"
}
/** Gets the truthiness (non emptyness) of the default of `p` if that value is mutable */
private boolean mutableDefaultValue(Parameter p) {
exists(Dict d | p.getDefault() = d |
exists(d.getAKey()) and result = true
or
not exists(d.getAKey()) and result = false
)
or
exists(List l | p.getDefault() = l |
exists(l.getAnElt()) and result = true
or
not exists(l.getAnElt()) and result = false
)
}
class NonEmptyMutableValue extends TaintKind {
NonEmptyMutableValue() { this = "non-empty mutable value" }
}
class EmptyMutableValue extends TaintKind {
EmptyMutableValue() { this = "empty mutable value" }
override boolean booleanValue() { result = false }
}
class MutableDefaultValue extends TaintSource {
boolean nonEmpty;
MutableDefaultValue() { nonEmpty = mutableDefaultValue(this.(NameNode).getNode()) }
override string toString() { result = "mutable default value" }
override predicate isSourceOf(TaintKind kind) {
nonEmpty = false and kind instanceof EmptyMutableValue
or
nonEmpty = true and kind instanceof NonEmptyMutableValue
}
}
private ClassValue mutable_class() {
result = Value::named("list") or
result = Value::named("dict")
}
class Mutation extends TaintSink {
Mutation() {
exists(AugAssign a | a.getTarget().getAFlowNode() = this)
or
exists(Call c, Attribute a | c.getFunc() = a |
a.getObject().getAFlowNode() = this and
not safe_method(a.getName()) and
this.(ControlFlowNode).pointsTo().getClass() = mutable_class()
)
}
override predicate sinks(TaintKind kind) {
kind instanceof EmptyMutableValue
or
kind instanceof NonEmptyMutableValue
}
}
from TaintedPathSource src, TaintedPathSink sink
where src.flowsTo(sink)
select sink.getSink(), src, sink, "$@ flows to here and is mutated.", src.getSource(),
from
ModificationOfParameterWithDefault::Configuration config, DataFlow::PathNode source,
DataFlow::PathNode sink
where config.hasFlowPath(source, sink)
select sink.getNode(), source, sink, "$@ flows to here and is mutated.", source.getNode(),
"Default value"

1
python/ql/src/Functions/SignatureOverriddenMethod.ql
View File

@@ -24,7 +24,6 @@ where
not derived.getScope().isSpecialMethod() and
derived.getName() != "__init__" and
derived.isNormalMethod() and
not derived.getScope().isSpecialMethod() and
// call to overrides distributed for efficiency
(
derived.overrides(base) and derived.minParameters() > base.maxParameters()

115
python/ql/src/Functions/SignatureSpecialMethods.ql
View File

@@ -13,98 +13,29 @@
import python
predicate is_unary_op(string name) {
name = "__del__" or
name = "__repr__" or
name = "__str__" or
name = "__hash__" or
name = "__bool__" or
name = "__nonzero__" or
name = "__unicode__" or
name = "__len__" or
name = "__iter__" or
name = "__reversed__" or
name = "__neg__" or
name = "__pos__" or
name = "__abs__" or
name = "__invert__" or
name = "__complex__" or
name = "__int__" or
name = "__float__" or
name = "__long__" or
name = "__oct__" or
name = "__hex__" or
name = "__index__" or
name = "__enter__"
name in [
"__del__", "__repr__", "__neg__", "__pos__", "__abs__", "__invert__", "__complex__",
"__int__", "__float__", "__long__", "__oct__", "__hex__", "__str__", "__index__", "__enter__",
"__hash__", "__bool__", "__nonzero__", "__unicode__", "__len__", "__iter__", "__reversed__"
]
}
predicate is_binary_op(string name) {
name = "__lt__" or
name = "__le__" or
name = "__eq__" or
name = "__ne__" or
name = "__gt__" or
name = "__ge__" or
name = "__cmp__" or
name = "__rcmp__" or
name = "__getattr___" or
name = "__getattribute___" or
name = "__delattr__" or
name = "__delete__" or
name = "__instancecheck__" or
name = "__subclasscheck__" or
name = "__getitem__" or
name = "__delitem__" or
name = "__contains__" or
name = "__add__" or
name = "__sub__" or
name = "__mul__" or
name = "__floordiv__" or
name = "__div__" or
name = "__truediv__" or
name = "__mod__" or
name = "__divmod__" or
name = "__lshift__" or
name = "__rshift__" or
name = "__and__" or
name = "__xor__" or
name = "__or__" or
name = "__radd__" or
name = "__rsub__" or
name = "__rmul__" or
name = "__rfloordiv__" or
name = "__rdiv__" or
name = "__rtruediv__" or
name = "__rmod__" or
name = "__rdivmod__" or
name = "__rpow__" or
name = "__rlshift__" or
name = "__rrshift__" or
name = "__rand__" or
name = "__rxor__" or
name = "__ror__" or
name = "__iadd__" or
name = "__isub__" or
name = "__imul__" or
name = "__ifloordiv__" or
name = "__idiv__" or
name = "__itruediv__" or
name = "__imod__" or
name = "__idivmod__" or
name = "__ipow__" or
name = "__ilshift__" or
name = "__irshift__" or
name = "__iand__" or
name = "__ixor__" or
name = "__ior__" or
name = "__coerce__"
name in [
"__lt__", "__le__", "__delattr__", "__delete__", "__instancecheck__", "__subclasscheck__",
"__getitem__", "__delitem__", "__contains__", "__add__", "__sub__", "__mul__", "__eq__",
"__floordiv__", "__div__", "__truediv__", "__mod__", "__divmod__", "__lshift__", "__rshift__",
"__and__", "__xor__", "__or__", "__ne__", "__radd__", "__rsub__", "__rmul__", "__rfloordiv__",
"__rdiv__", "__rtruediv__", "__rmod__", "__rdivmod__", "__rpow__", "__rlshift__", "__gt__",
"__rrshift__", "__rand__", "__rxor__", "__ror__", "__iadd__", "__isub__", "__imul__",
"__ifloordiv__", "__idiv__", "__itruediv__", "__ge__", "__imod__", "__idivmod__", "__ipow__",
"__ilshift__", "__irshift__", "__iand__", "__ixor__", "__ior__", "__coerce__", "__cmp__",
"__rcmp__", "__getattr___", "__getattribute___"
]
}
predicate is_ternary_op(string name) {
name = "__setattr__" or
name = "__set__" or
name = "__setitem__" or
name = "__getslice__" or
name = "__delslice__"
name in ["__setattr__", "__set__", "__setitem__", "__getslice__", "__delslice__"]
}
predicate is_quad_op(string name) { name = "__setslice__" or name = "__exit__" }
@@ -132,12 +63,12 @@ predicate incorrect_special_method_defn(
else
if required < func.minParameters()
then message = "Too many parameters" and show_counts = true
else
if func.minParameters() < required and not func.getScope().hasVarArg()
then
message = (required - func.minParameters()) + " default values(s) will never be used" and
show_counts = false
else none()
else (
func.minParameters() < required and
not func.getScope().hasVarArg() and
message = (required - func.minParameters()) + " default values(s) will never be used" and
show_counts = false
)
)
}

38
python/ql/src/Lexical/CommentedOutCode.qll
View File

@@ -197,7 +197,7 @@ class CommentedOutCodeBlock extends @py_comment {
* The location spans column `startcolumn` of line `startline` to
* column `endcolumn` of line `endline` in file `filepath`.
* For more information, see
* [Locations](https://help.semmle.com/QL/learn-ql/ql/locations.html).
* [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/).
*/
predicate hasLocationInfo(
string filepath, int startline, int startcolumn, int endline, int endcolumn
@@ -297,41 +297,17 @@ private predicate file_or_url(Comment c) {
c.getText().regexpMatch("#[^'\"]+(\\[a-zA-Z]\\w*)+\\.[a-zA-Z]+.*")
}
private string operator_keyword() {
result = "import" or
result = "and" or
result = "is" or
result = "or" or
result = "in" or
result = "not" or
result = "as"
}
private string operator_keyword() { result in ["import", "and", "is", "or", "in", "not", "as"] }
private string keyword_requiring_colon() {
result = "try" or
result = "while" or
result = "elif" or
result = "else" or
result = "if" or
result = "except" or
result = "def" or
result = "class"
result in ["try", "while", "elif", "else", "if", "except", "def", "class"]
}
private string other_keyword() {
result = "del" or
result = "lambda" or
result = "from" or
result = "global" or
result = "with" or
result = "assert" or
result = "yield" or
result = "finally" or
result = "print" or
result = "exec" or
result = "raise" or
result = "return" or
result = "for"
result in [
"del", "lambda", "raise", "return", "for", "from", "global", "with", "assert", "yield",
"finally", "print", "exec"
]
}
private string a_keyword() {

4
python/ql/src/Metrics/Internal/Extents.qll
View File

@@ -20,7 +20,7 @@ class RangeFunction extends Function {
* The location spans column `startcolumn` of line `startline` to
* column `endcolumn` of line `endline` in file `filepath`.
* For more information, see
* [Locations](https://help.semmle.com/QL/learn-ql/ql/locations.html).
* [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/).
*/
predicate hasLocationInfo(
string filepath, int startline, int startcolumn, int endline, int endcolumn
@@ -40,7 +40,7 @@ class RangeClass extends Class {
* The location spans column `startcolumn` of line `startline` to
* column `endcolumn` of line `endline` in file `filepath`.
* For more information, see
* [Locations](https://help.semmle.com/QL/learn-ql/ql/locations.html).
* [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/).
*/
predicate hasLocationInfo(
string filepath, int startline, int startcolumn, int endline, int endcolumn

8
python/ql/src/Security/CWE-089/SqlInjection.qhelp
View File

@@ -9,6 +9,13 @@ If a database query (such as a SQL or NoSQL query) is built from
user-provided data without sufficient sanitization, a user
may be able to run malicious database queries.
</p>
<p>
This also includes using the <code>TextClause</code> class in the
<code><a href="https://pypi.org/project/SQLAlchemy/">SQLAlchemy</a></code> PyPI package,
which is used to represent a literal SQL fragment and is inserted directly into the
final SQL when used in a query built using the ORM.
</p>
</overview>
<recommendation>
@@ -52,5 +59,6 @@ vulnerable to SQL injection attacks. In this example, if <code>username</code> w
<references>
<li>Wikipedia: <a href="https://en.wikipedia.org/wiki/SQL_injection">SQL injection</a>.</li>
<li>OWASP: <a href="https://cheatsheetseries.owasp.org/cheatsheets/SQL_Injection_Prevention_Cheat_Sheet.html">SQL Injection Prevention Cheat Sheet</a>.</li>
<li><a href="https://docs.sqlalchemy.org/en/14/core/sqlelement.html#sqlalchemy.sql.expression.text.params.text">SQLAlchemy documentation for TextClause</a>.</li>
</references>
</qhelp>

1
python/ql/src/Security/CWE-327/WeakSensitiveDataHashing.ql
View File

@@ -8,6 +8,7 @@
* @id py/weak-sensitive-data-hashing
* @tags security
* external/cwe/cwe-327
* external/cwe/cwe-328
* external/cwe/cwe-916
*/

0
python/ql/src/experimental/Security/CWE-730/RegexInjection.qhelp → python/ql/src/Security/CWE-730/RegexInjection.qhelp
View File

15
python/ql/src/experimental/Security/CWE-730/RegexInjection.ql → python/ql/src/Security/CWE-730/RegexInjection.ql
View File

@@ -5,25 +5,24 @@
* exponential time on certain inputs.
* @kind path-problem
* @problem.severity error
* @precision high
* @id py/regex-injection
* @tags security
* external/cwe/cwe-730
* external/cwe/cwe-400
*/
// determine precision above
import python
import experimental.semmle.python.security.injection.RegexInjection
private import semmle.python.Concepts
import semmle.python.security.injection.RegexInjection
import DataFlow::PathGraph
from
RegexInjectionFlowConfig config, DataFlow::PathNode source, DataFlow::PathNode sink,
RegexInjectionSink regexInjectionSink, Attribute methodAttribute
RegexInjection::Configuration config, DataFlow::PathNode source, DataFlow::PathNode sink,
RegexExecution regexExecution
where
config.hasFlowPath(source, sink) and
regexInjectionSink = sink.getNode() and
methodAttribute = regexInjectionSink.getRegexMethod()
regexExecution = sink.getNode().(RegexInjection::Sink).getRegexExecution()
select sink.getNode(), source, sink,
"$@ regular expression is constructed from a $@ and executed by $@.", sink.getNode(), "This",
source.getNode(), "user-provided value", methodAttribute,
regexInjectionSink.getRegexModule() + "." + methodAttribute.getName()
source.getNode(), "user-provided value", regexExecution, regexExecution.getName()

0
python/ql/src/experimental/Security/CWE-730/re_bad.py → python/ql/src/Security/CWE-730/re_bad.py
View File

0
python/ql/src/experimental/Security/CWE-730/re_good.py → python/ql/src/Security/CWE-730/re_good.py
View File

2
python/ql/src/Security/CWE-798/HardcodedCredentials.ql
View File

@@ -88,7 +88,7 @@ class CredentialSink extends TaintSink {
CredentialSink() {
exists(string name |
name.regexpMatch(getACredentialRegex()) and
not name.suffix(name.length() - 4) = "file"
not name.matches("%file")
|
any(FunctionValue func).getNamedArgumentForCall(_, name) = this
or

8
python/ql/src/Statements/ModificationOfLocals.ql
View File

@@ -30,5 +30,11 @@ predicate modification_of_locals(ControlFlowNode f) {
}
from AstNode a, ControlFlowNode f
where modification_of_locals(f) and a = f.getNode()
where
modification_of_locals(f) and
a = f.getNode() and
// in module level scope `locals() == globals()`
// see https://docs.python.org/3/library/functions.html#locals
// FP report in https://github.com/github/codeql/issues/6674
not a.getScope() instanceof ModuleScope
select a, "Modification of the locals() dictionary will have no effect on the local variables."

13
python/ql/src/Statements/SideEffectInAssert.ql
View File

@@ -15,16 +15,9 @@ import python
predicate func_with_side_effects(Expr e) {
exists(string name | name = e.(Attribute).getName() or name = e.(Name).getId() |
name = "print" or
name = "write" or
name = "append" or
name = "pop" or
name = "remove" or
name = "discard" or
name = "delete" or
name = "close" or
name = "open" or
name = "exit"
name in [
"print", "write", "append", "pop", "remove", "discard", "delete", "close", "open", "exit"
]
)
}

12
python/ql/src/Variables/UnusedLocalVariable.ql
View File

@@ -19,6 +19,7 @@ predicate unused_local(Name unused, LocalVariable v) {
def.getVariable() = v and
def.isUnused() and
not exists(def.getARedef()) and
not exists(annotation_without_assignment(v)) and
def.isRelevant() and
not v = any(Nonlocal n).getAVariable() and
not exists(def.getNode().getParentNode().(FunctionDef).getDefinedFunction().getADecorator()) and
@@ -26,6 +27,17 @@ predicate unused_local(Name unused, LocalVariable v) {
)
}
/**
* Gets any annotation of the local variable `v` that does not also reassign its value.
*
* TODO: This predicate should not be needed. Rather, annotated "assignments" that do not actually
* assign a value should not result in the creation of an SSA variable (which then goes unused).
*/
private AnnAssign annotation_without_assignment(LocalVariable v) {
result.getTarget() = v.getAStore() and
not exists(result.getValue())
}
from Name unused, LocalVariable v
where
unused_local(unused, v) and

2
python/ql/src/analysis/AlertSuppression.ql
View File

@@ -88,7 +88,7 @@ class SuppressionScope extends @py_comment {
* The location spans column `startcolumn` of line `startline` to
* column `endcolumn` of line `endline` in file `filepath`.
* For more information, see
* [Locations](https://help.semmle.com/QL/learn-ql/ql/locations.html).
* [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/).
*/
predicate hasLocationInfo(
string filepath, int startline, int startcolumn, int endline, int endcolumn

15
python/ql/src/analysis/Consistency.ql
View File

@@ -8,15 +8,10 @@ import python
import DefinitionTracking
predicate uniqueness_error(int number, string what, string problem) {
(
what = "toString" or
what = "getLocation" or
what = "getNode" or
what = "getDefinition" or
what = "getEntryNode" or
what = "getOrigin" or
what = "getAnInferredType"
) and
what in [
"toString", "getLocation", "getNode", "getDefinition", "getEntryNode", "getOrigin",
"getAnInferredType"
] and
(
number = 0 and problem = "no results for " + what + "()"
or
@@ -141,7 +136,7 @@ predicate builtin_object_consistency(string clsname, string problem, string what
or
not exists(o.toString()) and
problem = "no toString" and
not exists(string name | name.prefix(7) = "_semmle" | py_special_objects(o, name)) and
not exists(string name | name.matches("\\_semmle%") | py_special_objects(o, name)) and
not o = unknownValue()
)
}

2
python/ql/src/analysis/DefinitionTracking.qll
View File

@@ -477,7 +477,7 @@ class NiceLocationExpr extends @py_expr {
* The location spans column `bc` of line `bl` to
* column `ec` of line `el` in file `f`.
* For more information, see
* [Locations](https://help.semmle.com/QL/learn-ql/ql/locations.html).
* [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/).
*/
predicate hasLocationInfo(string f, int bl, int bc, int el, int ec) {
/* Attribute location for x.y is that of 'y' so that url does not overlap with that of 'x' */

4
python/ql/src/codeql-suites/python-code-scanning.qls
View File

@@ -1,4 +1,4 @@
- description: Standard Code Scanning queries for Python
- qlpack: codeql-python
- queries: .
- apply: code-scanning-selectors.yml
from: codeql-suite-helpers
from: codeql/suite-helpers

4
python/ql/src/codeql-suites/python-lgtm-full.qls
View File

@@ -1,7 +1,7 @@
- description: Standard LGTM queries for Python, including ones not displayed by default
- qlpack: codeql-python
- queries: .
- apply: lgtm-selectors.yml
from: codeql-suite-helpers
from: codeql/suite-helpers
# These are only for IDE use.
- exclude:
tags contain:

2
python/ql/src/codeql-suites/python-lgtm.qls
View File

@@ -1,4 +1,4 @@
- description: Standard LGTM queries for Python
- apply: codeql-suites/python-lgtm-full.qls
- apply: lgtm-displayed-only.yml
from: codeql-suite-helpers
from: codeql/suite-helpers

4
python/ql/src/codeql-suites/python-security-and-quality.qls
View File

@@ -1,4 +1,4 @@
- description: Security-and-quality queries for Python
- qlpack: codeql-python
- queries: .
- apply: security-and-quality-selectors.yml
from: codeql-suite-helpers
from: codeql/suite-helpers

4
python/ql/src/codeql-suites/python-security-extended.qls
View File

@@ -1,4 +1,4 @@
- description: Security-extended queries for Python
- qlpack: codeql-python
- queries: .
- apply: security-extended-selectors.yml
from: codeql-suite-helpers
from: codeql/suite-helpers

6
python/ql/src/default.qll
View File

@@ -1,6 +0,0 @@
/**
* WARNING: Use of this module is DEPRECATED.
* All new queries should use `import python`.
*/
import python

49
python/ql/src/experimental/Security/CWE-117/LogInjection.qhelp Normal file
View File

@@ -0,0 +1,49 @@
<!DOCTYPE qhelp PUBLIC
"-//Semmle//qhelp//EN"
"qhelp.dtd">
<qhelp>
<overview>
<p>If unsanitized user input is written to a log entry, a malicious user may be able to forge new log entries.</p>
<p>Forgery can occur if a user provides some input creating the appearance of multiple
log entries. This can include unescaped new-line characters, or HTML or other markup.</p>
</overview>
<recommendation>
<p>
User input should be suitably sanitized before it is logged.
</p>
<p>
If the log entries are plain text then line breaks should be removed from user input, using for example
<code>replace(old, new)</code> or similar. Care should also be taken that user input is clearly marked
in log entries, and that a malicious user cannot cause confusion in other ways.
</p>
<p>
For log entries that will be displayed in HTML, user input should be HTML encoded before being logged, to prevent forgery and
other forms of HTML injection.
</p>
</recommendation>
<example>
<p>
In the example, the name provided by the user is recorded using the log output function (<code>logging.info</code> or <code>app.logger.info</code>, etc.).
In these four cases, the name provided by the user is not provided The processing is recorded. If a malicious user provides <code>Guest%0D%0AUser name: Admin</code>
as a parameter, the log entry will be divided into two lines, the first line is <code>User name: Guest</code> code>, the second line is <code>User name: Admin</code>.
</p>
<sample src="LogInjectionBad.py" />
<p>
In a good example, the program uses the <code>replace</code> function to provide parameter processing to the user, and replace <code>\r\n</code> and <code>\n</code>
with empty characters. To a certain extent, the occurrence of log injection vulnerabilities is reduced.
</p>
<sample src="LogInjectionGood.py" />
</example>
<references>
<li>OWASP: <a href="https://owasp.org/www-community/attacks/Log_Injection">Log Injection</a>.</li>
</references>
</qhelp>

20
python/ql/src/experimental/Security/CWE-117/LogInjection.ql Normal file
View File

@@ -0,0 +1,20 @@
/**
* @name Log Injection
* @description Building log entries from user-controlled data is vulnerable to
* insertion of forged log entries by a malicious user.
* @kind path-problem
* @problem.severity error
* @precision high
* @id py/log-injection
* @tags security
* external/cwe/cwe-117
*/
import python
import experimental.semmle.python.security.injection.LogInjection
import DataFlow::PathGraph
from LogInjectionFlowConfig config, DataFlow::PathNode source, DataFlow::PathNode sink
where config.hasFlowPath(source, sink)
select sink.getNode(), source, sink, "$@ flows to log entry.", source.getNode(),
"User-provided value"

44
python/ql/src/experimental/Security/CWE-117/LogInjectionBad.py Normal file
View File

@@ -0,0 +1,44 @@
#!/usr/bin/env python
# -*- coding: UTF-8 -*-
"""
@Desc ：Log Injection
"""
from flask import Flask
from flask import request
from django.utils.log import request_logger
import logging
logging.basicConfig(level=logging.DEBUG)
app = Flask(__name__)
@app.route('/bad1')
def bad1():
name = request.args.get('name')
app.logger.info('User name: ' + name) # Bad
return 'bad1'
@app.route('/bad2')
def bad2():
name = request.args.get('name')
logging.info('User name: ' + name) # Bad
return 'bad2'
@app.route('/bad3')
def bad3():
name = request.args.get('name')
request_logger.warn('User name: ' + name) # Bad
return 'bad3'
@app.route('/bad4')
def bad4():
name = request.args.get('name')
logtest = logging.getLogger('test')
logtest.debug('User name: ' + name) # Bad
return 'bad4'
if __name__ == '__main__':
app.debug = True
handler = logging.FileHandler('log')
app.logger.addHandler(handler)
app.run()

25
python/ql/src/experimental/Security/CWE-117/LogInjectionGood.py Normal file
View File

@@ -0,0 +1,25 @@
#!/usr/bin/env python
# -*- coding: UTF-8 -*-
"""
@Desc ：Log Injection
"""
from flask import Flask
from flask import request
import logging
logging.basicConfig(level=logging.DEBUG)
app = Flask(__name__)
@app.route('/good1')
def good1():
name = request.args.get('name')
name = name.replace('\r\n','').replace('\n','')
logging.info('User name: ' + name) # Good
return 'good1'
if __name__ == '__main__':
app.debug = True
handler = logging.FileHandler('log')
app.logger.addHandler(handler)
app.run()

35
python/ql/src/experimental/Security/CWE-348/ClientSuppliedIpUsedInSecurityCheck.py Normal file
View File

@@ -0,0 +1,35 @@
#!/usr/bin/env python
# -*- coding: UTF-8 -*-
"""
@Desc ：ip address spoofing
"""
from flask import Flask
from flask import request
app = Flask(__name__)
@app.route('/bad1')
def bad1():
client_ip = request.headers.get('x-forwarded-for')
if not client_ip.startswith('192.168.'):
raise Exception('ip illegal')
return 'bad1'
@app.route('/bad2')
def bad2():
client_ip = request.headers.get('x-forwarded-for')
if not client_ip == '127.0.0.1':
raise Exception('ip illegal')
return 'bad2'
@app.route('/good1')
def good1():
client_ip = request.headers.get('x-forwarded-for')
client_ip = client_ip.split(',')[client_ip.split(',').length - 1]
if not client_ip == '127.0.0.1':
raise Exception('ip illegal')
return 'good1'
if __name__ == '__main__':
app.debug = True
app.run()

35
python/ql/src/experimental/Security/CWE-348/ClientSuppliedIpUsedInSecurityCheck.qhelp Normal file
View File

@@ -0,0 +1,35 @@
<!DOCTYPE qhelp PUBLIC
"-//Semmle//qhelp//EN"
"qhelp.dtd">
<qhelp>
<overview>
<p>An original client IP address is retrieved from an http header (<code>X-Forwarded-For</code> or <code>X-Real-IP</code> or <code>Proxy-Client-IP</code>
etc.), which is used to ensure security. Attackers can forge the value of these identifiers to
bypass a ban-list, for example.</p>
</overview>
<recommendation>
<p>Do not trust the values of HTTP headers allegedly identifying the originating IP. If you are aware your application will run behind some reverse proxies then the last entry of a <code>X-Forwarded-For</code> header value may be more trustworthy than the rest of it because some reverse proxies append the IP address they observed to the end of any remote-supplied header.</p>
</recommendation>
<example>
<p>The following examples show the bad case and the good case respectively.
In <code>bad1</code> method and <code>bad2</code> method, the client ip the <code>X-Forwarded-For</code> is split into comma-separated values, but the less-trustworthy first one is used. Both of these examples could be deceived by providing a forged HTTP header. The method
<code>good1</code> similarly splits an <code>X-Forwarded-For</code> value, but uses the last, more-trustworthy entry.</p>
<sample src="ClientSuppliedIpUsedInSecurityCheck.py" />
</example>
<references>
<li>Dennis Schneider: <a href="https://www.dennis-schneider.com/blog/prevent-ip-address-spoofing-with-x-forwarded-for-header-and-aws-elb-in-clojure-ring/">
Prevent IP address spoofing with X-Forwarded-For header when using AWS ELB and Clojure Ring</a>
</li>
<li>Security Rule Zero: <a href="https://www.f5.com/company/blog/security-rule-zero-a-warning-about-x-forwarded-for">A Warning about X-Forwarded-For</a>
</li>
</references>
</qhelp>

56
python/ql/src/experimental/Security/CWE-348/ClientSuppliedIpUsedInSecurityCheck.ql Normal file
View File

@@ -0,0 +1,56 @@
/**
* @name IP address spoofing
* @description A remote endpoint identifier is read from an HTTP header. Attackers can modify the value
* of the identifier to forge the client ip.
* @kind path-problem
* @problem.severity error
* @precision high
* @id py/ip-address-spoofing
* @tags security
* external/cwe/cwe-348
*/
import python
import semmle.python.dataflow.new.DataFlow
import semmle.python.dataflow.new.TaintTracking
import semmle.python.ApiGraphs
import ClientSuppliedIpUsedInSecurityCheckLib
import DataFlow::PathGraph
/**
* Taint-tracking configuration tracing flow from obtaining a client ip from an HTTP header to a sensitive use.
*/
class ClientSuppliedIpUsedInSecurityCheckConfig extends TaintTracking::Configuration {
ClientSuppliedIpUsedInSecurityCheckConfig() { this = "ClientSuppliedIpUsedInSecurityCheckConfig" }
override predicate isSource(DataFlow::Node source) {
source instanceof ClientSuppliedIpUsedInSecurityCheck
}
override predicate isSink(DataFlow::Node sink) { sink instanceof PossibleSecurityCheck }
override predicate isAdditionalTaintStep(DataFlow::Node pred, DataFlow::Node succ) {
exists(DataFlow::CallCfgNode ccn |
ccn = API::moduleImport("netaddr").getMember("IPAddress").getACall() and
ccn.getArg(0) = pred and
ccn = succ
)
}
override predicate isSanitizer(DataFlow::Node node) {
// `client_supplied_ip.split(",")[n]` for `n` > 0
exists(Subscript ss |
not ss.getIndex().(IntegerLiteral).getText() = "0" and
ss.getObject().(Call).getFunc().(Attribute).getName() = "split" and
ss.getObject().(Call).getAnArg().(StrConst).getText() = "," and
ss = node.asExpr()
)
}
}
from
ClientSuppliedIpUsedInSecurityCheckConfig config, DataFlow::PathNode source,
DataFlow::PathNode sink
where config.hasFlowPath(source, sink)
select sink.getNode(), source, sink, "IP address spoofing might include code from $@.",
source.getNode(), "this user input"

152
python/ql/src/experimental/Security/CWE-348/ClientSuppliedIpUsedInSecurityCheckLib.qll Normal file
View File

@@ -0,0 +1,152 @@
private import python
private import semmle.python.Concepts
private import semmle.python.ApiGraphs
private import semmle.python.dataflow.new.DataFlow
private import semmle.python.dataflow.new.RemoteFlowSources
/**
* A data flow source of the client ip obtained according to the remote endpoint identifier specified
* (`X-Forwarded-For`, `X-Real-IP`, `Proxy-Client-IP`, etc.) in the header.
*
* For example: `request.headers.get("X-Forwarded-For")`.
*/
abstract class ClientSuppliedIpUsedInSecurityCheck extends DataFlow::CallCfgNode { }
private class FlaskClientSuppliedIpUsedInSecurityCheck extends ClientSuppliedIpUsedInSecurityCheck {
FlaskClientSuppliedIpUsedInSecurityCheck() {
exists(RemoteFlowSource rfs, DataFlow::AttrRead get |
rfs.getSourceType() = "flask.request" and this.getFunction() = get
|
// `get` is a call to request.headers.get or request.headers.get_all or request.headers.getlist
// request.headers
get.getObject()
.(DataFlow::AttrRead)
// request
.getObject()
.getALocalSource() = rfs and
get.getAttributeName() in ["get", "get_all", "getlist"] and
get.getObject().(DataFlow::AttrRead).getAttributeName() = "headers" and
this.getArg(0).asExpr().(StrConst).getText().toLowerCase() = clientIpParameterName()
)
}
}
private class DjangoClientSuppliedIpUsedInSecurityCheck extends ClientSuppliedIpUsedInSecurityCheck {
DjangoClientSuppliedIpUsedInSecurityCheck() {
exists(RemoteFlowSource rfs, DataFlow::AttrRead get |
rfs.getSourceType() = "django.http.request.HttpRequest" and this.getFunction() = get
|
// `get` is a call to request.headers.get or request.META.get
// request.headers
get.getObject()
.(DataFlow::AttrRead)
// request
.getObject()
.getALocalSource() = rfs and
get.getAttributeName() = "get" and
get.getObject().(DataFlow::AttrRead).getAttributeName() in ["headers", "META"] and
this.getArg(0).asExpr().(StrConst).getText().toLowerCase() = clientIpParameterName()
)
}
}
private class TornadoClientSuppliedIpUsedInSecurityCheck extends ClientSuppliedIpUsedInSecurityCheck {
TornadoClientSuppliedIpUsedInSecurityCheck() {
exists(RemoteFlowSource rfs, DataFlow::AttrRead get |
rfs.getSourceType() = "tornado.web.RequestHandler" and this.getFunction() = get
|
// `get` is a call to `rfs`.request.headers.get
// `rfs`.request.headers
get.getObject()
.(DataFlow::AttrRead)
// `rfs`.request
.getObject()
.(DataFlow::AttrRead)
// `rfs`
.getObject()
.getALocalSource() = rfs and
get.getAttributeName() in ["get", "get_list"] and
get.getObject().(DataFlow::AttrRead).getAttributeName() = "headers" and
this.getArg(0).asExpr().(StrConst).getText().toLowerCase() = clientIpParameterName()
)
}
}
private string clientIpParameterName() {
result in [
"x-forwarded-for", "x_forwarded_for", "x-real-ip", "x_real_ip", "proxy-client-ip",
"proxy_client_ip", "wl-proxy-client-ip", "wl_proxy_client_ip", "http_x_forwarded_for",
"http-x-forwarded-for", "http_x_forwarded", "http_x_cluster_client_ip", "http_client_ip",
"http_forwarded_for", "http_forwarded", "http_via", "remote_addr"
]
}
/** A data flow sink for ip address forgery vulnerabilities. */
abstract class PossibleSecurityCheck extends DataFlow::Node { }
/** A data flow sink for sql operation. */
private class SqlOperationAsSecurityCheck extends PossibleSecurityCheck {
SqlOperationAsSecurityCheck() { this = any(SqlExecution e).getSql() }
}
/**
* A data flow sink for remote client ip comparison.
*
* For example: `if not ipAddr.startswith('192.168.') : ...` determine whether the client ip starts
* with `192.168.`, and the program can be deceived by forging the ip address.
*/
private class CompareSink extends PossibleSecurityCheck {
CompareSink() {
exists(Call call |
call.getFunc().(Attribute).getName() = "startswith" and
call.getArg(0).(StrConst).getText().regexpMatch(getIpAddressRegex()) and
not call.getArg(0).(StrConst).getText() = "0:0:0:0:0:0:0:1" and
call.getFunc().(Attribute).getObject() = this.asExpr()
)
or
exists(Compare compare |
(
compare.getOp(0) instanceof Eq or
compare.getOp(0) instanceof NotEq
) and
(
compare.getLeft() = this.asExpr() and
compare.getComparator(0).(StrConst).getText() instanceof PrivateHostName and
not compare.getComparator(0).(StrConst).getText() = "0:0:0:0:0:0:0:1"
or
compare.getComparator(0) = this.asExpr() and
compare.getLeft().(StrConst).getText() instanceof PrivateHostName and
not compare.getLeft().(StrConst).getText() = "0:0:0:0:0:0:0:1"
)
)
or
exists(Compare compare |
(
compare.getOp(0) instanceof In or
compare.getOp(0) instanceof NotIn
) and
(
compare.getLeft() = this.asExpr()
or
compare.getComparator(0) = this.asExpr() and
not compare.getLeft().(StrConst).getText() in ["%", ",", "."]
)
)
}
}
string getIpAddressRegex() {
result =
"^((10\\.((1\\d{2})?|(2[0-4]\\d)?|(25[0-5])?|([1-9]\\d|[0-9])?)(\\.)?)|(192\\.168\\.)|172\\.(1[6789]|2[0-9]|3[01])\\.)((1\\d{2})?|(2[0-4]\\d)?|(25[0-5])?|([1-9]\\d|[0-9])?)(\\.)?((1\\d{2})?|(2[0-4]\\d)?|(25[0-5])?|([1-9]\\d|[0-9])?)$"
}
/**
* A string matching private host names of IPv4 and IPv6, which only matches the host portion therefore checking for port is not necessary.
* Several examples are localhost, reserved IPv4 IP addresses including 127.0.0.1, 10.x.x.x, 172.16.x,x, 192.168.x,x, and reserved IPv6 addresses including [0:0:0:0:0:0:0:1] and [::1]
*/
private class PrivateHostName extends string {
bindingset[this]
PrivateHostName() {
this.regexpMatch("(?i)localhost(?:[:/?#].*)?|127\\.0\\.0\\.1(?:[:/?#].*)?|10(?:\\.[0-9]+){3}(?:[:/?#].*)?|172\\.16(?:\\.[0-9]+){2}(?:[:/?#].*)?|192.168(?:\\.[0-9]+){2}(?:[:/?#].*)?|\\[?0:0:0:0:0:0:0:1\\]?(?:[:/?#].*)?|\\[?::1\\]?(?:[:/?#].*)?")
}
}

23
python/ql/src/experimental/Security/CWE-522/LDAPInsecureAuth.qhelp Normal file
View File

@@ -0,0 +1,23 @@
<!DOCTYPE qhelp PUBLIC
"-//Semmle//qhelp//EN"
"qhelp.dtd">
<qhelp>
<overview>
<p>Failing to ensure the utilization of SSL in an LDAP connection can cause the entire communication
to be sent in cleartext making it easier for an attacker to intercept it.</p>
</overview>
<recommendation>
<p>Always set <code>use_SSL</code> to <code>True</code>, call <code>start_tls_s()</code> or set a proper option flag (<code>ldap.OPT_X_TLS_XXXXXX</code>).</p>
</recommendation>
<example>
<p>This example shows both good and bad ways to deal with this issue under Python 3.</p>
<p>The first one sets <code>use_SSL</code> to true as a keyword argument whereas the second one fails to provide a value for it, so
the default one is used (<code>False</code>).</p>
<sample src="examples/LDAPInsecureAuth.py" />
</example>
</qhelp>

21
python/ql/src/experimental/Security/CWE-522/LDAPInsecureAuth.ql Normal file
View File

@@ -0,0 +1,21 @@
/**
* @name Python Insecure LDAP Authentication
* @description Python LDAP Insecure LDAP Authentication
* @kind path-problem
* @problem.severity error
* @id py/insecure-ldap-auth
* @tags experimental
* security
* external/cwe/cwe-522
* external/cwe/cwe-523
*/
// determine precision above
import python
import DataFlow::PathGraph
import experimental.semmle.python.security.LDAPInsecureAuth
from LDAPInsecureAuthConfig config, DataFlow::PathNode source, DataFlow::PathNode sink
where config.hasFlowPath(source, sink)
select sink.getNode(), source, sink, "$@ is authenticated insecurely.", sink.getNode(),
"This LDAP host"

20
python/ql/src/experimental/Security/CWE-522/examples/LDAPInsecureAuth.py Normal file
View File

@@ -0,0 +1,20 @@
from ldap3 import Server, Connection, ALL
from flask import request, Flask
app = Flask(__name__)
@app.route("/good")
def good():
srv = Server(host, port, use_ssl=True)
conn = Connection(srv, dn, password)
conn.search(dn, search_filter)
return conn.response
@app.route("/bad")
def bad():
srv = Server(host, port)
conn = Connection(srv, dn, password)
conn.search(dn, search_filter)
return conn.response

0
python/ql/src/experimental/Security/CWE-643/xpath.qhelp → python/ql/src/experimental/Security/CWE-643/XpathInjection.qhelp
View File

33
python/ql/src/experimental/Security/CWE-643/XpathInjection.ql Normal file
View File

@@ -0,0 +1,33 @@
/**
* @name XPath query built from user-controlled sources
* @description Building a XPath query from user-controlled sources is vulnerable to insertion of
* malicious Xpath code by the user.
* @kind path-problem
* @problem.severity error
* @precision high
* @id py/xpath-injection
* @tags security
* external/cwe/cwe-643
*/
private import python
private import semmle.python.Concepts
private import semmle.python.dataflow.new.TaintTracking
private import semmle.python.Concepts
private import semmle.python.ApiGraphs
private import semmle.python.dataflow.new.RemoteFlowSources
private import semmle.python.dataflow.new.BarrierGuards
import XpathInjection::XpathInjection
import DataFlow::PathGraph
class XpathInjectionConfiguration extends TaintTracking::Configuration {
XpathInjectionConfiguration() { this = "PathNotNormalizedConfiguration" }
override predicate isSource(DataFlow::Node source) { source instanceof Source }
override predicate isSink(DataFlow::Node sink) { sink instanceof Sink }
}
from XpathInjectionConfiguration config, DataFlow::PathNode source, DataFlow::PathNode sink
where config.hasFlowPath(source, sink)
select sink, source, sink, "This Xpath query depends on $@.", source, "a user-provided value"

16
python/ql/src/semmle/python/security/dataflow/PolynomialReDoS.qll → python/ql/src/experimental/Security/CWE-643/XpathInjection.qll
View File

@@ -1,9 +1,9 @@
/**
* Provides a taint-tracking configuration for detecting "polynomial regular expression denial of service (ReDoS)" vulnerabilities.
* Provides a taint-tracking configuration for detecting "Xpath Injection" vulnerabilities.
*
* Note, for performance reasons: only import this file if
* `PolynomialReDoS::Configuration` is needed, otherwise
* `PolynomialReDoSCustomizations` should be imported instead.
* `XpathInjection::Configuration` is needed, otherwise
* `XpathInjectionCustomizations` should be imported instead.
*/
private import python
@@ -11,16 +11,16 @@ import semmle.python.dataflow.new.DataFlow
import semmle.python.dataflow.new.TaintTracking
/**
* Provides a taint-tracking configuration for detecting "polynomial regular expression denial of service (ReDoS)" vulnerabilities.
* Provides a taint-tracking configuration for detecting "Xpath Injection" vulnerabilities.
*/
module PolynomialReDoS {
import PolynomialReDoSCustomizations::PolynomialReDoS
module XpathInjection {
import XpathInjectionCustomizations::XpathInjection
/**
* A taint-tracking configuration for detecting "polynomial regular expression denial of service (ReDoS)" vulnerabilities.
* A taint-tracking configuration for detecting "Xpath Injection" vulnerabilities.
*/
class Configuration extends TaintTracking::Configuration {
Configuration() { this = "PolynomialReDoS" }
Configuration() { this = "Xpath Injection" }
override predicate isSource(DataFlow::Node source) { source instanceof Source }

105
python/ql/src/experimental/Security/CWE-643/XpathInjectionCustomizations.qll Normal file
View File

@@ -0,0 +1,105 @@
/**
* Provides class and predicates to track external data that
* may represent malicious xpath query objects.
*
* This module is intended to be imported into a taint-tracking query.
*/
private import python
private import semmle.python.Concepts
private import semmle.python.dataflow.new.TaintTracking
private import semmle.python.Concepts
private import semmle.python.ApiGraphs
private import semmle.python.dataflow.new.RemoteFlowSources
private import semmle.python.dataflow.new.BarrierGuards
/** Models Xpath Injection related classes and functions */
module XpathInjection {
/**
* A data flow source for "XPath injection" vulnerabilities.
*/
abstract class Source extends DataFlow::Node { }
/**
* A data flow sink for "XPath injection" vulnerabilities.
*/
abstract class Sink extends DataFlow::Node { }
/**
* A sanitizer for "XPath injection" vulnerabilities.
*/
abstract class Sanitizer extends DataFlow::Node { }
/**
* A sanitizer guard for "XPath injection" vulnerabilities.
*/
abstract class SanitizerGuard extends DataFlow::BarrierGuard { }
/**
* A source of remote user input, considered as a flow source.
*/
class RemoteFlowSourceAsSource extends Source, RemoteFlowSource { }
/** Returns an API node referring to `lxml.etree` */
API::Node etree() { result = API::moduleImport("lxml").getMember("etree") }
/** Returns an API node referring to `lxml.etree` */
API::Node etreeFromString() { result = etree().getMember("fromstring") }
/** Returns an API node referring to `lxml.etree.parse` */
API::Node etreeParse() { result = etree().getMember("parse") }
/** Returns an API node referring to `lxml.etree.parse` */
API::Node libxml2parseFile() { result = API::moduleImport("libxml2").getMember("parseFile") }
/**
* A Sink representing an argument to `etree.XPath` or `etree.ETXPath` call.
*
* from lxml import etree
* root = etree.XML("<xmlContent>")
* find_text = etree.XPath("`sink`")
* find_text = etree.ETXPath("`sink`")
*/
private class EtreeXpathArgument extends Sink {
EtreeXpathArgument() { this = etree().getMember(["XPath", "ETXPath"]).getACall().getArg(0) }
}
/**
* A Sink representing an argument to the `etree.XPath` call.
*
* from lxml import etree
* root = etree.fromstring(file(XML_DB).read(), XMLParser())
* find_text = root.xpath("`sink`")
*/
private class EtreeFromstringXpathArgument extends Sink {
EtreeFromstringXpathArgument() {
this = etreeFromString().getReturn().getMember("xpath").getACall().getArg(0)
}
}
/**
* A Sink representing an argument to the `xpath` call to a parsed xml document.
*
* from lxml import etree
* from io import StringIO
* f = StringIO('<foo><bar></bar></foo>')
* tree = etree.parse(f)
* r = tree.xpath('`sink`')
*/
private class ParseXpathArgument extends Sink {
ParseXpathArgument() { this = etreeParse().getReturn().getMember("xpath").getACall().getArg(0) }
}
/**
* A Sink representing an argument to the `xpathEval` call to a parsed libxml2 document.
*
* import libxml2
* tree = libxml2.parseFile("file.xml")
* r = tree.xpathEval('`sink`')
*/
private class ParseFileXpathEvalArgument extends Sink {
ParseFileXpathEvalArgument() {
this = libxml2parseFile().getReturn().getMember("xpathEval").getACall().getArg(0)
}
}
}

36
python/ql/src/experimental/Security/CWE-643/xpath.ql
View File

@@ -1,36 +0,0 @@
/**
* @name XPath query built from user-controlled sources
* @description Building a XPath query from user-controlled sources is vulnerable to insertion of
* malicious Xpath code by the user.
* @kind path-problem
* @problem.severity error
* @precision high
* @id py/xpath-injection
* @tags security
* external/cwe/cwe-643
*/
import python
import semmle.python.security.Paths
import semmle.python.security.strings.Untrusted
/* Sources */
import semmle.python.web.HttpRequest
/* Sinks */
import experimental.semmle.python.security.injection.Xpath
class XpathInjectionConfiguration extends TaintTracking::Configuration {
XpathInjectionConfiguration() { this = "Xpath injection configuration" }
override predicate isSource(TaintTracking::Source source) {
source instanceof HttpRequestTaintSource
}
override predicate isSink(TaintTracking::Sink sink) {
sink instanceof XpathInjection::XpathInjectionSink
}
}
from XpathInjectionConfiguration config, TaintedPathSource src, TaintedPathSink sink
where config.hasFlowPath(src, sink)
select sink.getSink(), src, sink, "This Xpath query depends on $@.", src.getSource(),
"a user-provided value"

40
python/ql/src/experimental/Security/CWE-943/NoSQLInjection.qhelp Normal file
View File

@@ -0,0 +1,40 @@
<!DOCTYPE qhelp PUBLIC
"-//Semmle//qhelp//EN"
"qhelp.dtd">
<qhelp>
<overview>
<p>
Passing user-controlled sources into NoSQL queries can result in a NoSQL injection flaw.
This tainted NoSQL query containing a user-controlled source can then execute a malicious query in a NoSQL database such as MongoDB.
In order for the user-controlled source to taint the NoSQL query, the user-controller source must be converted into a Python object using something like <code>json.loads</code> or <code>xmltodict.parse</code>.
</p>
<p>
Because a user-controlled source is passed into the query, the malicious user can have complete control over the query itself.
When the tainted query is executed, the malicious user can commit malicious actions such as bypassing role restrictions or accessing and modifying restricted data in the NoSQL database.
</p>
</overview>
<recommendation>
<p>
NoSQL injections can be prevented by escaping user-input's special characters that are passed into the NoSQL query from the user-supplied source.
Alternatively, using a sanitize library such as MongoSanitizer will ensure that user-supplied sources can not act as a malicious query.
</p>
</recommendation>
<example>
<p>In the example below, the user-supplied source is passed to a MongoDB function that queries the MongoDB database.</p>
<sample src="examples/NoSQLInjection-bad.py" />
<p> This can be fixed by using a sanitizer library like MongoSanitizer as shown in this annotated code version below.</p>
<sample src="examples/NoSQLInjection-good.py" />
</example>
<references>
<li>Mongoengine: <a href="http://mongoengine.org/">Documentation</a>.</li>
<li>Flask-Mongoengine: <a href="http://docs.mongoengine.org/projects/flask-mongoengine/en/latest/">Documentation</a>.</li>
<li>PyMongo: <a href="https://pypi.org/project/pymongo/">Documentation</a>.</li>
<li>Flask-PyMongo: <a href="https://flask-pymongo.readthedocs.io/en/latest/">Documentation</a>.</li>
<li>OWASP: <a href="https://owasp.org/www-pdf-archive/GOD16-NOSQL.pdf">NoSQL Injection</a>.</li>
<li>Security Stack Exchange Discussion: <a href="https://security.stackexchange.com/questions/83231/mongodb-nosql-injection-in-python-code">Question 83231</a>.</li>
</references>
</qhelp>

19
python/ql/src/experimental/Security/CWE-943/NoSQLInjection.ql Normal file
View File

@@ -0,0 +1,19 @@
/**
* @name NoSQL Injection
* @description Building a NoSQL query from user-controlled sources is vulnerable to insertion of
* malicious NoSQL code by the user.
* @kind path-problem
* @problem.severity error
* @id py/nosql-injection
* @tags experimental
* security
* external/cwe/cwe-943
*/
import python
import experimental.semmle.python.security.injection.NoSQLInjection
from CustomPathNode source, CustomPathNode sink
where noSQLInjectionFlow(source, sink)
select sink, source, sink, "$@ NoSQL query contains an unsanitized $@", sink, "This", source,
"user-provided value"

13
python/ql/src/experimental/Security/CWE-943/examples/NoSQLInjection-bad.py Normal file
View File

@@ -0,0 +1,13 @@
from flask import Flask, request
from flask_pymongo import PyMongo
import json
mongo = PyMongo(app)
@app.route("/")
def home_page():
unsanitized_search = request.args['search']
json_search = json.loads(unsanitized_search)
result = mongo.db.user.find({'name': json_search})

15
python/ql/src/experimental/Security/CWE-943/examples/NoSQLInjection-good.py Normal file
View File

@@ -0,0 +1,15 @@
from flask import Flask, request
from flask_pymongo import PyMongo
from mongosanitizer.sanitizer import sanitize
import json
mongo = PyMongo(app)
@app.route("/")
def home_page():
unsafe_search = request.args['search']
json_search = json.loads(unsafe_search)
safe_search = sanitize(unsanitized_search)
result = client.db.collection.find_one({'data': safe_search})

226
python/ql/src/experimental/semmle/python/Concepts.qll
View File

@@ -14,71 +14,34 @@ private import semmle.python.dataflow.new.RemoteFlowSources
private import semmle.python.dataflow.new.TaintTracking
private import experimental.semmle.python.Frameworks
/** Provides classes for modeling Regular Expression-related APIs. */
module RegexExecution {
/** Provides classes for modeling log related APIs. */
module LogOutput {
/**
* A data-flow node that executes a regular expression.
* A data flow node for log output.
*
* Extend this class to model new APIs. If you want to refine existing API models,
* extend `RegexExecution` instead.
* extend `LogOutput` instead.
*/
abstract class Range extends DataFlow::Node {
/**
* Gets the argument containing the executed expression.
* Get the parameter value of the log output function.
*/
abstract DataFlow::Node getRegexNode();
/**
* Gets the library used to execute the regular expression.
*/
abstract string getRegexModule();
abstract DataFlow::Node getAnInput();
}
}
/**
* A data-flow node that executes a regular expression.
* A data flow node for log output.
*
* Extend this class to refine existing API models. If you want to model new APIs,
* extend `RegexExecution::Range` instead.
* extend `LogOutput::Range` instead.
*/
class RegexExecution extends DataFlow::Node {
RegexExecution::Range range;
class LogOutput extends DataFlow::Node {
LogOutput::Range range;
RegexExecution() { this = range }
LogOutput() { this = range }
DataFlow::Node getRegexNode() { result = range.getRegexNode() }
string getRegexModule() { result = range.getRegexModule() }
}
/** Provides classes for modeling Regular Expression escape-related APIs. */
module RegexEscape {
/**
* A data-flow node that escapes a regular expression.
*
* Extend this class to model new APIs. If you want to refine existing API models,
* extend `RegexEscape` instead.
*/
abstract class Range extends DataFlow::Node {
/**
* Gets the argument containing the escaped expression.
*/
abstract DataFlow::Node getRegexNode();
}
}
/**
* A data-flow node that escapes a regular expression.
*
* Extend this class to refine existing API models. If you want to model new APIs,
* extend `RegexEscape::Range` instead.
*/
class RegexEscape extends DataFlow::Node {
RegexEscape::Range range;
RegexEscape() { this = range }
DataFlow::Node getRegexNode() { result = range.getRegexNode() }
DataFlow::Node getAnInput() { result = range.getAnInput() }
}
/** Provides classes for modeling LDAP query execution-related APIs. */
@@ -156,10 +119,20 @@ module LDAPBind {
* extend `LDAPBind` instead.
*/
abstract class Range extends DataFlow::Node {
/**
* Gets the argument containing the binding host.
*/
abstract DataFlow::Node getHost();
/**
* Gets the argument containing the binding expression.
*/
abstract DataFlow::Node getPassword();
/**
* Holds if the binding process use SSL.
*/
abstract predicate useSSL();
}
}
@@ -174,7 +147,20 @@ class LDAPBind extends DataFlow::Node {
LDAPBind() { this = range }
/**
* Gets the argument containing the binding host.
*/
DataFlow::Node getHost() { result = range.getHost() }
/**
* Gets the argument containing the binding expression.
*/
DataFlow::Node getPassword() { result = range.getPassword() }
/**
* Holds if the binding process use SSL.
*/
predicate useSSL() { range.useSSL() }
}
/** Provides classes for modeling SQL sanitization libraries. */
@@ -210,6 +196,64 @@ class SQLEscape extends DataFlow::Node {
DataFlow::Node getAnInput() { result = range.getAnInput() }
}
/** Provides a class for modeling NoSQL execution APIs. */
module NoSQLQuery {
/**
* A data-flow node that executes NoSQL queries.
*
* Extend this class to model new APIs. If you want to refine existing API models,
* extend `NoSQLQuery` instead.
*/
abstract class Range extends DataFlow::Node {
/** Gets the argument that specifies the NoSQL query to be executed. */
abstract DataFlow::Node getQuery();
}
}
/**
* A data-flow node that executes NoSQL queries.
*
* Extend this class to refine existing API models. If you want to model new APIs,
* extend `NoSQLQuery::Range` instead.
*/
class NoSQLQuery extends DataFlow::Node {
NoSQLQuery::Range range;
NoSQLQuery() { this = range }
/** Gets the argument that specifies the NoSQL query to be executed. */
DataFlow::Node getQuery() { result = range.getQuery() }
}
/** Provides classes for modeling NoSQL sanitization-related APIs. */
module NoSQLSanitizer {
/**
* A data-flow node that collects functions sanitizing NoSQL queries.
*
* Extend this class to model new APIs. If you want to refine existing API models,
* extend `NoSQLSanitizer` instead.
*/
abstract class Range extends DataFlow::Node {
/** Gets the argument that specifies the NoSQL query to be sanitized. */
abstract DataFlow::Node getAnInput();
}
}
/**
* A data-flow node that collects functions sanitizing NoSQL queries.
*
* Extend this class to model new APIs. If you want to refine existing API models,
* extend `NoSQLSanitizer::Range` instead.
*/
class NoSQLSanitizer extends DataFlow::Node {
NoSQLSanitizer::Range range;
NoSQLSanitizer() { this = range }
/** Gets the argument that specifies the NoSQL query to be sanitized. */
DataFlow::Node getAnInput() { result = range.getAnInput() }
}
/** Provides classes for modeling HTTP Header APIs. */
module HeaderDeclaration {
/**
@@ -234,8 +278,8 @@ module HeaderDeclaration {
/**
* A data-flow node that collects functions setting HTTP Headers.
*
* Extend this class to model new APIs. If you want to refine existing API models,
* extend `HeaderDeclaration` instead.
* Extend this class to refine existing API models. If you want to model new APIs,
* extend `HeaderDeclaration::Range` instead.
*/
class HeaderDeclaration extends DataFlow::Node {
HeaderDeclaration::Range range;
@@ -248,65 +292,63 @@ class HeaderDeclaration extends DataFlow::Node {
DataFlow::Node getNameArg() { result = range.getNameArg() }
/**
* Gets the argument containing the header name.
* Gets the argument containing the header value.
*/
DataFlow::Node getValueArg() { result = range.getValueArg() }
}
module ExperimentalHTTP {
/**
* A data-flow node that sets a cookie in an HTTP response.
*
* Extend this class to refine existing API models. If you want to model new APIs,
* extend `HTTP::CookieWrite::Range` instead.
*/
class CookieWrite extends DataFlow::Node {
CookieWrite::Range range;
CookieWrite() { this = range }
/**
* Gets the argument, if any, specifying the raw cookie header.
*/
DataFlow::Node getHeaderArg() { result = range.getHeaderArg() }
/**
* Gets the argument, if any, specifying the cookie name.
*/
DataFlow::Node getNameArg() { result = range.getNameArg() }
/**
* Gets the argument, if any, specifying the cookie value.
*/
DataFlow::Node getValueArg() { result = range.getValueArg() }
}
/** Provides a class for modeling new cookie writes on HTTP responses. */
module CookieWrite {
/**
* A data-flow node that sets a cookie in an HTTP response.
*
* Extend this class to refine existing API models. If you want to model new APIs,
* extend `HTTP::CookieWrite::Range` instead.
* Note: we don't require that this redirect must be sent to a client (a kind of
* "if a tree falls in a forest and nobody hears it" situation).
*
* Extend this class to model new APIs. If you want to refine existing API models,
* extend `HttpResponse` instead.
*/
class CookieWrite extends DataFlow::Node {
CookieWrite::Range range;
CookieWrite() { this = range }
abstract class Range extends DataFlow::Node {
/**
* Gets the argument, if any, specifying the raw cookie header.
*/
DataFlow::Node getHeaderArg() { result = range.getHeaderArg() }
abstract DataFlow::Node getHeaderArg();
/**
* Gets the argument, if any, specifying the cookie name.
*/
DataFlow::Node getNameArg() { result = range.getNameArg() }
abstract DataFlow::Node getNameArg();
/**
* Gets the argument, if any, specifying the cookie value.
*/
DataFlow::Node getValueArg() { result = range.getValueArg() }
}
/** Provides a class for modeling new cookie writes on HTTP responses. */
module CookieWrite {
/**
* A data-flow node that sets a cookie in an HTTP response.
*
* Note: we don't require that this redirect must be sent to a client (a kind of
* "if a tree falls in a forest and nobody hears it" situation).
*
* Extend this class to model new APIs. If you want to refine existing API models,
* extend `HttpResponse` instead.
*/
abstract class Range extends DataFlow::Node {
/**
* Gets the argument, if any, specifying the raw cookie header.
*/
abstract DataFlow::Node getHeaderArg();
/**
* Gets the argument, if any, specifying the cookie name.
*/
abstract DataFlow::Node getNameArg();
/**
* Gets the argument, if any, specifying the cookie value.
*/
abstract DataFlow::Node getValueArg();
}
abstract DataFlow::Node getValueArg();
}
}

2
python/ql/src/experimental/semmle/python/Frameworks.qll
View File

@@ -7,3 +7,5 @@ private import experimental.semmle.python.frameworks.Flask
private import experimental.semmle.python.frameworks.Django
private import experimental.semmle.python.frameworks.Werkzeug
private import experimental.semmle.python.frameworks.LDAP
private import experimental.semmle.python.frameworks.NoSQL
private import experimental.semmle.python.frameworks.Log

17
python/ql/src/experimental/semmle/python/frameworks/Django.qll
View File

@@ -8,16 +8,27 @@ private import semmle.python.frameworks.Django
private import semmle.python.dataflow.new.DataFlow
private import experimental.semmle.python.Concepts
private import semmle.python.ApiGraphs
import semmle.python.dataflow.new.RemoteFlowSources
private module PrivateDjango {
API::Node django() { result = API::moduleImport("django") }
private module django {
API::Node http() { result = django().getMember("http") }
API::Node http() { result = API::moduleImport("django").getMember("http") }
module http {
API::Node response() { result = http().getMember("response") }
API::Node request() { result = http().getMember("request") }
module request {
module HttpRequest {
class DjangoGETParameter extends DataFlow::Node, RemoteFlowSource::Range {
DjangoGETParameter() { this = request().getMember("GET").getMember("get").getACall() }
override string getSourceType() { result = "django.http.request.GET.get" }
}
}
}
module response {
module HttpResponse {
API::Node baseClassRef() {

68
python/ql/src/experimental/semmle/python/frameworks/LDAP.qll
View File

@@ -88,6 +88,11 @@ private module LDAP {
result.(DataFlow::AttrRead).getAttributeName() instanceof LDAP2BindMethods
}
/**List of SSL-demanding options */
private class LDAPSSLOptions extends DataFlow::Node {
LDAPSSLOptions() { this = ldap().getMember("OPT_X_TLS_" + ["DEMAND", "HARD"]).getAUse() }
}
/**
* A class to find `ldap` methods binding a connection.
*
@@ -99,6 +104,44 @@ private module LDAP {
override DataFlow::Node getPassword() {
result in [this.getArg(1), this.getArgByName("cred")]
}
override DataFlow::Node getHost() {
exists(DataFlow::CallCfgNode initialize |
this.getFunction().(DataFlow::AttrRead).getObject().getALocalSource() = initialize and
initialize = ldapInitialize().getACall() and
result = initialize.getArg(0)
)
}
override predicate useSSL() {
// use initialize to correlate `this` and so avoid FP in several instances
exists(DataFlow::CallCfgNode initialize |
// ldap.set_option(ldap.OPT_X_TLS_%s)
ldap().getMember("set_option").getACall().getArg(_) instanceof LDAPSSLOptions
or
this.getFunction().(DataFlow::AttrRead).getObject().getALocalSource() = initialize and
initialize = ldapInitialize().getACall() and
(
// ldap_connection.start_tls_s()
// see https://www.python-ldap.org/en/python-ldap-3.3.0/reference/ldap.html#ldap.LDAPObject.start_tls_s
exists(DataFlow::MethodCallNode startTLS |
startTLS.getObject().getALocalSource() = initialize and
startTLS.getMethodName() = "start_tls_s"
)
or
// ldap_connection.set_option(ldap.OPT_X_TLS_%s, True)
exists(DataFlow::CallCfgNode setOption |
setOption.getFunction().(DataFlow::AttrRead).getObject().getALocalSource() =
initialize and
setOption.getFunction().(DataFlow::AttrRead).getAttributeName() = "set_option" and
setOption.getArg(0) instanceof LDAPSSLOptions and
not DataFlow::exprNode(any(False falseExpr))
.(DataFlow::LocalSourceNode)
.flowsTo(setOption.getArg(1))
)
)
)
}
}
/**
@@ -166,6 +209,31 @@ private module LDAP {
override DataFlow::Node getPassword() {
result in [this.getArg(2), this.getArgByName("password")]
}
override DataFlow::Node getHost() {
exists(DataFlow::CallCfgNode serverCall |
serverCall = ldap3Server().getACall() and
this.getArg(0).getALocalSource() = serverCall and
result = serverCall.getArg(0)
)
}
override predicate useSSL() {
exists(DataFlow::CallCfgNode serverCall |
serverCall = ldap3Server().getACall() and
this.getArg(0).getALocalSource() = serverCall and
DataFlow::exprNode(any(True trueExpr))
.(DataFlow::LocalSourceNode)
.flowsTo([serverCall.getArg(2), serverCall.getArgByName("use_ssl")])
)
or
// ldap_connection.start_tls_s()
// see https://www.python-ldap.org/en/python-ldap-3.3.0/reference/ldap.html#ldap.LDAPObject.start_tls_s
exists(DataFlow::MethodCallNode startTLS |
startTLS.getMethodName() = "start_tls_s" and
startTLS.getObject().getALocalSource() = this
)
}
}
/**

118
python/ql/src/experimental/semmle/python/frameworks/Log.qll Normal file
View File

@@ -0,0 +1,118 @@
/**
* Provides classes modeling security-relevant aspects of the log libraries.
*/
private import python
private import semmle.python.dataflow.new.DataFlow
private import semmle.python.dataflow.new.TaintTracking
private import semmle.python.dataflow.new.RemoteFlowSources
private import experimental.semmle.python.Concepts
private import semmle.python.frameworks.Flask
private import semmle.python.ApiGraphs
/**
* Provides models for Python's log-related libraries.
*/
private module log {
/**
* Log output method list.
*
* See https://docs.python.org/3/library/logging.html#logger-objects
*/
private class LogOutputMethods extends string {
LogOutputMethods() {
this in ["info", "error", "warn", "warning", "debug", "critical", "exception", "log"]
}
}
/**
* The class used to find the log output method of the `logging` module.
*
* See `LogOutputMethods`
*/
private class LoggingCall extends DataFlow::CallCfgNode, LogOutput::Range {
LoggingCall() {
this = API::moduleImport("logging").getMember(any(LogOutputMethods m)).getACall()
}
override DataFlow::Node getAnInput() {
this.getFunction().(DataFlow::AttrRead).getAttributeName() != "log" and
result in [this.getArg(_), this.getArgByName(_)] // this includes the arg named "msg"
or
this.getFunction().(DataFlow::AttrRead).getAttributeName() = "log" and
result in [this.getArg(any(int i | i > 0)), this.getArgByName(any(string s | s != "level"))]
}
}
/**
* The class used to find log output methods related to the `logging.getLogger` instance.
*
* See `LogOutputMethods`
*/
private class LoggerCall extends DataFlow::CallCfgNode, LogOutput::Range {
LoggerCall() {
this =
API::moduleImport("logging")
.getMember("getLogger")
.getReturn()
.getMember(any(LogOutputMethods m))
.getACall()
}
override DataFlow::Node getAnInput() {
this.getFunction().(DataFlow::AttrRead).getAttributeName() != "log" and
result in [this.getArg(_), this.getArgByName(_)] // this includes the arg named "msg"
or
this.getFunction().(DataFlow::AttrRead).getAttributeName() = "log" and
result in [this.getArg(any(int i | i > 0)), this.getArgByName(any(string s | s != "level"))]
}
}
/**
* The class used to find the relevant log output method of the `flask.Flask.logger` instance (flask application).
*
* See `LogOutputMethods`
*/
private class FlaskLoggingCall extends DataFlow::CallCfgNode, LogOutput::Range {
FlaskLoggingCall() {
this =
Flask::FlaskApp::instance()
.getMember("logger")
.getMember(any(LogOutputMethods m))
.getACall()
}
override DataFlow::Node getAnInput() {
this.getFunction().(DataFlow::AttrRead).getAttributeName() != "log" and
result in [this.getArg(_), this.getArgByName(_)] // this includes the arg named "msg"
or
this.getFunction().(DataFlow::AttrRead).getAttributeName() = "log" and
result in [this.getArg(any(int i | i > 0)), this.getArgByName(any(string s | s != "level"))]
}
}
/**
* The class used to find the relevant log output method of the `django.utils.log.request_logger` instance (django application).
*
* See `LogOutputMethods`
*/
private class DjangoLoggingCall extends DataFlow::CallCfgNode, LogOutput::Range {
DjangoLoggingCall() {
this =
API::moduleImport("django")
.getMember("utils")
.getMember("log")
.getMember("request_logger")
.getMember(any(LogOutputMethods m))
.getACall()
}
override DataFlow::Node getAnInput() {
this.getFunction().(DataFlow::AttrRead).getAttributeName() != "log" and
result in [this.getArg(_), this.getArgByName(_)] // this includes the arg named "msg"
or
this.getFunction().(DataFlow::AttrRead).getAttributeName() = "log" and
result in [this.getArg(any(int i | i > 0)), this.getArgByName(any(string s | s != "level"))]
}
}
}

215
python/ql/src/experimental/semmle/python/frameworks/NoSQL.qll Normal file
View File

@@ -0,0 +1,215 @@
/**
* Provides classes modeling security-relevant aspects of the standard libraries.
* Note: some modeling is done internally in the dataflow/taint tracking implementation.
*/
private import python
private import semmle.python.dataflow.new.DataFlow
private import semmle.python.dataflow.new.TaintTracking
private import semmle.python.dataflow.new.RemoteFlowSources
private import experimental.semmle.python.Concepts
private import semmle.python.ApiGraphs
private module NoSQL {
// API Nodes returning `Mongo` instances.
/** Gets a reference to `pymongo.MongoClient` */
private API::Node pyMongo() {
result = API::moduleImport("pymongo").getMember("MongoClient").getReturn()
}
/** Gets a reference to `flask_pymongo.PyMongo` */
private API::Node flask_PyMongo() {
result = API::moduleImport("flask_pymongo").getMember("PyMongo").getReturn()
}
/** Gets a reference to `mongoengine` */
private API::Node mongoEngine() { result = API::moduleImport("mongoengine") }
/** Gets a reference to `flask_mongoengine.MongoEngine` */
private API::Node flask_MongoEngine() {
result = API::moduleImport("flask_mongoengine").getMember("MongoEngine").getReturn()
}
/**
* Gets a reference to an initialized `Mongo` instance.
* See `pyMongo()`, `flask_PyMongo()`
*/
private API::Node mongoInstance() {
result = pyMongo() or
result = flask_PyMongo()
}
/**
* Gets a reference to an initialized `Mongo` DB instance.
* See `mongoEngine()`, `flask_MongoEngine()`
*/
private API::Node mongoDBInstance() {
result = mongoEngine().getMember(["get_db", "connect"]).getReturn() or
result = mongoEngine().getMember("connection").getMember(["get_db", "connect"]).getReturn() or
result = flask_MongoEngine().getMember("get_db").getReturn()
}
/**
* Gets a reference to a `Mongo` DB use.
*
* See `mongoInstance()`, `mongoDBInstance()`.
*/
private DataFlow::LocalSourceNode mongoDB(DataFlow::TypeTracker t) {
t.start() and
(
exists(SubscriptNode subscript |
subscript.getObject() = mongoInstance().getAUse().asCfgNode() and
result.asCfgNode() = subscript
)
or
result.(DataFlow::AttrRead).getObject() = mongoInstance().getAUse()
or
result = mongoDBInstance().getAUse()
)
or
exists(DataFlow::TypeTracker t2 | result = mongoDB(t2).track(t2, t))
}
/**
* Gets a reference to a `Mongo` DB use.
*
* ```py
* from flask_pymongo import PyMongo
* mongo = PyMongo(app)
* mongo.db.user.find({'name': safe_search})
* ```
*
* `mongo.db` would be a use of a `Mongo` instance, and so the result.
*/
private DataFlow::Node mongoDB() { mongoDB(DataFlow::TypeTracker::end()).flowsTo(result) }
/**
* Gets a reference to a `Mongo` collection use.
*
* See `mongoDB()`.
*/
private DataFlow::LocalSourceNode mongoCollection(DataFlow::TypeTracker t) {
t.start() and
(
exists(SubscriptNode subscript | result.asCfgNode() = subscript |
subscript.getObject() = mongoDB().asCfgNode()
)
or
result.(DataFlow::AttrRead).getObject() = mongoDB()
)
or
exists(DataFlow::TypeTracker t2 | result = mongoCollection(t2).track(t2, t))
}
/**
* Gets a reference to a `Mongo` collection use.
*
* ```py
* from flask_pymongo import PyMongo
* mongo = PyMongo(app)
* mongo.db.user.find({'name': safe_search})
* ```
*
* `mongo.db.user` would be a use of a `Mongo` collection, and so the result.
*/
private DataFlow::Node mongoCollection() {
mongoCollection(DataFlow::TypeTracker::end()).flowsTo(result)
}
/** This class represents names of find_* relevant `Mongo` collection-level operation methods. */
private class MongoCollectionMethodNames extends string {
MongoCollectionMethodNames() {
this in [
"find", "find_raw_batches", "find_one", "find_one_and_delete", "find_and_modify",
"find_one_and_replace", "find_one_and_update", "find_one_or_404"
]
}
}
/**
* Gets a reference to a `Mongo` collection method.
*
* ```py
* from flask_pymongo import PyMongo
* mongo = PyMongo(app)
* mongo.db.user.find({'name': safe_search})
* ```
*
* `mongo.db.user.find` would be a collection method, and so the result.
*/
private DataFlow::Node mongoCollectionMethod() {
mongoCollection() = result.(DataFlow::AttrRead).getObject() and
result.(DataFlow::AttrRead).getAttributeName() instanceof MongoCollectionMethodNames
}
/**
* Gets a reference to a `Mongo` collection method call
*
* ```py
* from flask_pymongo import PyMongo
* mongo = PyMongo(app)
* mongo.db.user.find({'name': safe_search})
* ```
*
* `mongo.db.user.find({'name': safe_search})` would be a collection method call, and so the result.
*/
private class MongoCollectionCall extends DataFlow::CallCfgNode, NoSQLQuery::Range {
MongoCollectionCall() { this.getFunction() = mongoCollectionMethod() }
override DataFlow::Node getQuery() { result = this.getArg(0) }
}
/**
* Gets a reference to a call from a class whose base is a reference to `mongoEngine()` or `flask_MongoEngine()`'s
* `Document` or `EmbeddedDocument` objects and its attribute is `objects`.
*
* ```py
* from flask_mongoengine import MongoEngine
* db = MongoEngine(app)
* class Movie(db.Document):
* title = db.StringField(required=True)
*
* Movie.objects(__raw__=json_search)
* ```
*
* `Movie.objects(__raw__=json_search)` would be the result.
*/
private class MongoEngineObjectsCall extends DataFlow::CallCfgNode, NoSQLQuery::Range {
MongoEngineObjectsCall() {
this =
[mongoEngine(), flask_MongoEngine()]
.getMember(["Document", "EmbeddedDocument"])
.getASubclass()
.getMember("objects")
.getACall()
}
override DataFlow::Node getQuery() { result = this.getArgByName(_) }
}
/** Gets a reference to `mongosanitizer.sanitizer.sanitize` */
private class MongoSanitizerCall extends DataFlow::CallCfgNode, NoSQLSanitizer::Range {
MongoSanitizerCall() {
this =
API::moduleImport("mongosanitizer").getMember("sanitizer").getMember("sanitize").getACall()
}
override DataFlow::Node getAnInput() { result = this.getArg(0) }
}
/**
* ObjectId returns a string representing an id.
* If at any time ObjectId can't parse it's input (like when a tainted dict in passed in),
* then ObjectId will throw an error preventing the query from running.
*/
private class BsonObjectIdCall extends DataFlow::CallCfgNode, NoSQLSanitizer::Range {
BsonObjectIdCall() {
this =
API::moduleImport(["bson", "bson.objectid", "bson.json_util"])
.getMember("ObjectId")
.getACall()
}
override DataFlow::Node getAnInput() { result = this.getArg(0) }
}
}

148
python/ql/src/experimental/semmle/python/frameworks/SqlAlchemy.qll
View File

@@ -1,148 +0,0 @@
/**
* Provides classes modeling security-relevant aspects of the 'SqlAlchemy' package.
* See https://pypi.org/project/SQLAlchemy/.
*/
private import python
private import semmle.python.dataflow.new.DataFlow
private import semmle.python.dataflow.new.TaintTracking
private import semmle.python.ApiGraphs
private import semmle.python.Concepts
private import experimental.semmle.python.Concepts
private module SqlAlchemy {
/**
* Returns an instantization of a SqlAlchemy Session object.
* See https://docs.sqlalchemy.org/en/14/orm/session_api.html#sqlalchemy.orm.Session and
* https://docs.sqlalchemy.org/en/14/orm/session_api.html#sqlalchemy.orm.sessionmaker
*/
private API::Node getSqlAlchemySessionInstance() {
result = API::moduleImport("sqlalchemy.orm").getMember("Session").getReturn() or
result = API::moduleImport("sqlalchemy.orm").getMember("sessionmaker").getReturn().getReturn()
}
/**
* Returns an instantization of a SqlAlchemy Engine object.
* See https://docs.sqlalchemy.org/en/14/core/engines.html#sqlalchemy.create_engine
*/
private API::Node getSqlAlchemyEngineInstance() {
result = API::moduleImport("sqlalchemy").getMember("create_engine").getReturn()
}
/**
* Returns an instantization of a SqlAlchemy Query object.
* See https://docs.sqlalchemy.org/en/14/orm/query.html?highlight=query#sqlalchemy.orm.Query
*/
private API::Node getSqlAlchemyQueryInstance() {
result = getSqlAlchemySessionInstance().getMember("query").getReturn()
}
/**
* A call to `execute` meant to execute an SQL expression
* See the following links:
* - https://docs.sqlalchemy.org/en/14/core/connections.html?highlight=execute#sqlalchemy.engine.Connection.execute
* - https://docs.sqlalchemy.org/en/14/core/connections.html?highlight=execute#sqlalchemy.engine.Engine.execute
* - https://docs.sqlalchemy.org/en/14/orm/session_api.html?highlight=execute#sqlalchemy.orm.Session.execute
*/
private class SqlAlchemyExecuteCall extends DataFlow::CallCfgNode, SqlExecution::Range {
SqlAlchemyExecuteCall() {
// new way
this = getSqlAlchemySessionInstance().getMember("execute").getACall() or
this =
getSqlAlchemyEngineInstance()
.getMember(["connect", "begin"])
.getReturn()
.getMember("execute")
.getACall()
}
override DataFlow::Node getSql() { result = this.getArg(0) }
}
/**
* A call to `scalar` meant to execute an SQL expression
* See https://docs.sqlalchemy.org/en/14/orm/session_api.html#sqlalchemy.orm.Session.scalar and
* https://docs.sqlalchemy.org/en/14/core/connections.html?highlight=scalar#sqlalchemy.engine.Engine.scalar
*/
private class SqlAlchemyScalarCall extends DataFlow::CallCfgNode, SqlExecution::Range {
SqlAlchemyScalarCall() {
this =
[getSqlAlchemySessionInstance(), getSqlAlchemyEngineInstance()]
.getMember("scalar")
.getACall()
}
override DataFlow::Node getSql() { result = this.getArg(0) }
}
/**
* A call on a Query object
* See https://docs.sqlalchemy.org/en/14/orm/query.html?highlight=query#sqlalchemy.orm.Query
*/
private class SqlAlchemyQueryCall extends DataFlow::CallCfgNode, SqlExecution::Range {
SqlAlchemyQueryCall() {
this =
getSqlAlchemyQueryInstance()
.getMember(any(SqlAlchemyVulnerableMethodNames methodName))
.getACall()
}
override DataFlow::Node getSql() { result = this.getArg(0) }
}
/**
* This class represents a list of methods vulnerable to sql injection.
*
* See https://github.com/jty-team/codeql/pull/2#issue-611592361
*/
private class SqlAlchemyVulnerableMethodNames extends string {
SqlAlchemyVulnerableMethodNames() { this in ["filter", "filter_by", "group_by", "order_by"] }
}
/**
* Additional taint-steps for `sqlalchemy.text()`
*
* See https://docs.sqlalchemy.org/en/14/core/sqlelement.html#sqlalchemy.sql.expression.text
* See https://docs.sqlalchemy.org/en/14/core/sqlelement.html#sqlalchemy.sql.expression.TextClause
*/
class SqlAlchemyTextAdditionalTaintSteps extends TaintTracking::AdditionalTaintStep {
override predicate step(DataFlow::Node nodeFrom, DataFlow::Node nodeTo) {
exists(DataFlow::CallCfgNode call |
(
call = API::moduleImport("sqlalchemy").getMember("text").getACall()
or
call = API::moduleImport("sqlalchemy").getMember("sql").getMember("text").getACall()
or
call =
API::moduleImport("sqlalchemy")
.getMember("sql")
.getMember("expression")
.getMember("text")
.getACall()
or
call =
API::moduleImport("sqlalchemy")
.getMember("sql")
.getMember("expression")
.getMember("TextClause")
.getACall()
) and
nodeFrom in [call.getArg(0), call.getArgByName("text")] and
nodeTo = call
)
}
}
/**
* Gets a reference to `sqlescapy.sqlescape`.
*
* See https://pypi.org/project/sqlescapy/
*/
class SQLEscapySanitizerCall extends DataFlow::CallCfgNode, SQLEscape::Range {
SQLEscapySanitizerCall() {
this = API::moduleImport("sqlescapy").getMember("sqlescape").getACall()
}
override DataFlow::Node getAnInput() { result = this.getArg(0) }
}
}

88
python/ql/src/experimental/semmle/python/frameworks/Stdlib.qll
View File

@@ -9,91 +9,3 @@ private import semmle.python.dataflow.new.TaintTracking
private import semmle.python.dataflow.new.RemoteFlowSources
private import experimental.semmle.python.Concepts
private import semmle.python.ApiGraphs
/**
* Provides models for Python's `re` library.
*
* See https://docs.python.org/3/library/re.html
*/
private module Re {
/**
* List of `re` methods immediately executing an expression.
*
* See https://docs.python.org/3/library/re.html#module-contents
*/
private class RegexExecutionMethods extends string {
RegexExecutionMethods() {
this in ["match", "fullmatch", "search", "split", "findall", "finditer", "sub", "subn"]
}
}
/**
* A class to find `re` methods immediately executing an expression.
*
* See `RegexExecutionMethods`
*/
private class DirectRegex extends DataFlow::CallCfgNode, RegexExecution::Range {
DataFlow::Node regexNode;
DirectRegex() {
this = API::moduleImport("re").getMember(any(RegexExecutionMethods m)).getACall() and
regexNode = this.getArg(0)
}
override DataFlow::Node getRegexNode() { result = regexNode }
override string getRegexModule() { result = "re" }
}
/**
* A class to find `re` methods immediately executing a compiled expression by `re.compile`.
*
* Given the following example:
*
* ```py
* pattern = re.compile(input)
* pattern.match(s)
* ```
*
* This class will identify that `re.compile` compiles `input` and afterwards
* executes `re`'s `match`. As a result, `this` will refer to `pattern.match(s)`
* and `this.getRegexNode()` will return the node for `input` (`re.compile`'s first argument)
*
*
* See `RegexExecutionMethods`
*
* See https://docs.python.org/3/library/re.html#regular-expression-objects
*/
private class CompiledRegex extends DataFlow::MethodCallNode, RegexExecution::Range {
DataFlow::Node regexNode;
CompiledRegex() {
exists(DataFlow::MethodCallNode patternCall |
patternCall = API::moduleImport("re").getMember("compile").getACall() and
patternCall.flowsTo(this.getObject()) and
this.getMethodName() instanceof RegexExecutionMethods and
regexNode = patternCall.getArg(0)
)
}
override DataFlow::Node getRegexNode() { result = regexNode }
override string getRegexModule() { result = "re" }
}
/**
* A class to find `re` methods escaping an expression.
*
* See https://docs.python.org/3/library/re.html#re.escape
*/
class ReEscape extends DataFlow::CallCfgNode, RegexEscape::Range {
DataFlow::Node regexNode;
ReEscape() {
this = API::moduleImport("re").getMember("escape").getACall() and
regexNode = this.getArg(0)
}
override DataFlow::Node getRegexNode() { result = regexNode }
}
}

34
python/ql/src/experimental/semmle/python/frameworks/Xmltodict.qll Normal file
View File

@@ -0,0 +1,34 @@
/**
* Provides classes modeling security-relevant aspects of the `xmltodict` PyPI package.
* See https://pypi.org/project/xmltodict/
*/
private import python
private import semmle.python.dataflow.new.DataFlow
private import semmle.python.Concepts
private import semmle.python.ApiGraphs
/**
* Provides models for the `xmltodict` PyPI package.
* See https://pypi.org/project/xmltodict/
*/
private module XmlToDictModel {
/** Gets a reference to the `xmltodict` module. */
API::Node xmltodict() { result = API::moduleImport("xmltodict") }
/**
* A call to `xmltodict.parse`
* See https://github.com/martinblech/xmltodict/blob/ae19c452ca000bf243bfc16274c060bf3bf7cf51/xmltodict.py#L198
*/
private class XmlToDictParseCall extends Decoding::Range, DataFlow::CallCfgNode {
XmlToDictParseCall() { this = xmltodict().getMember("parse").getACall() }
override predicate mayExecuteInput() { none() }
override DataFlow::Node getAnInput() { result = this.getArg(0) }
override DataFlow::Node getOutput() { result = this }
override string getFormat() { result = "XML" }
}
}

106
python/ql/src/experimental/semmle/python/security/LDAPInsecureAuth.qll Normal file
View File

@@ -0,0 +1,106 @@
/**
* Provides a taint-tracking configuration for detecting LDAP injection vulnerabilities
*/
import python
import semmle.python.dataflow.new.DataFlow
import semmle.python.dataflow.new.TaintTracking
import semmle.python.dataflow.new.RemoteFlowSources
import experimental.semmle.python.Concepts
string getFullHostRegex() { result = "(?i)ldap://.+" }
string getSchemaRegex() { result = "(?i)ldap(://)?" }
string getPrivateHostRegex() {
result =
"(?i)localhost(?:[:/?#].*)?|127\\.0\\.0\\.1(?:[:/?#].*)?|10(?:\\.[0-9]+){3}(?:[:/?#].*)?|172\\.16(?:\\.[0-9]+){2}(?:[:/?#].*)?|192.168(?:\\.[0-9]+){2}(?:[:/?#].*)?|\\[?0:0:0:0:0:0:0:1\\]?(?:[:/?#].*)?|\\[?::1\\]?(?:[:/?#].*)?"
}
// "ldap://somethingon.theinternet.com"
class LDAPFullHost extends StrConst {
LDAPFullHost() {
exists(string s |
s = this.getText() and
s.regexpMatch(getFullHostRegex()) and
// check what comes after the `ldap://` prefix
not s.substring(7, s.length()).regexpMatch(getPrivateHostRegex())
)
}
}
class LDAPSchema extends StrConst {
LDAPSchema() { this.getText().regexpMatch(getSchemaRegex()) }
}
class LDAPPrivateHost extends StrConst {
LDAPPrivateHost() { this.getText().regexpMatch(getPrivateHostRegex()) }
}
predicate concatAndCompareAgainstFullHostRegex(LDAPSchema schema, StrConst host) {
not host instanceof LDAPPrivateHost and
(schema.getText() + host.getText()).regexpMatch(getFullHostRegex())
}
// "ldap://" + "somethingon.theinternet.com"
class LDAPBothStrings extends BinaryExpr {
LDAPBothStrings() { concatAndCompareAgainstFullHostRegex(this.getLeft(), this.getRight()) }
}
// schema + host
class LDAPBothVar extends BinaryExpr {
LDAPBothVar() {
exists(SsaVariable schemaVar, SsaVariable hostVar |
this.getLeft() = schemaVar.getVariable().getALoad() and // getAUse is incompatible with Expr
this.getRight() = hostVar.getVariable().getALoad() and
concatAndCompareAgainstFullHostRegex(schemaVar
.getDefinition()
.getImmediateDominator()
.getNode(), hostVar.getDefinition().getImmediateDominator().getNode())
)
}
}
// schema + "somethingon.theinternet.com"
class LDAPVarString extends BinaryExpr {
LDAPVarString() {
exists(SsaVariable schemaVar |
this.getLeft() = schemaVar.getVariable().getALoad() and
concatAndCompareAgainstFullHostRegex(schemaVar
.getDefinition()
.getImmediateDominator()
.getNode(), this.getRight())
)
}
}
// "ldap://" + host
class LDAPStringVar extends BinaryExpr {
LDAPStringVar() {
exists(SsaVariable hostVar |
this.getRight() = hostVar.getVariable().getALoad() and
concatAndCompareAgainstFullHostRegex(this.getLeft(),
hostVar.getDefinition().getImmediateDominator().getNode())
)
}
}
/**
* A taint-tracking configuration for detecting LDAP insecure authentications.
*/
class LDAPInsecureAuthConfig extends TaintTracking::Configuration {
LDAPInsecureAuthConfig() { this = "LDAPInsecureAuthConfig" }
override predicate isSource(DataFlow::Node source) {
source instanceof RemoteFlowSource or
source.asExpr() instanceof LDAPFullHost or
source.asExpr() instanceof LDAPBothStrings or
source.asExpr() instanceof LDAPBothVar or
source.asExpr() instanceof LDAPVarString or
source.asExpr() instanceof LDAPStringVar
}
override predicate isSink(DataFlow::Node sink) {
exists(LDAPBind ldapBind | not ldapBind.useSSL() and sink = ldapBind.getHost())
}
}

24
python/ql/src/experimental/semmle/python/security/injection/LogInjection.qll Normal file
View File

@@ -0,0 +1,24 @@
import python
import semmle.python.Concepts
import experimental.semmle.python.Concepts
import semmle.python.dataflow.new.DataFlow
import semmle.python.dataflow.new.TaintTracking
import semmle.python.dataflow.new.RemoteFlowSources
/**
* A taint-tracking configuration for tracking untrusted user input used in log entries.
*/
class LogInjectionFlowConfig extends TaintTracking::Configuration {
LogInjectionFlowConfig() { this = "LogInjectionFlowConfig" }
override predicate isSource(DataFlow::Node source) { source instanceof RemoteFlowSource }
override predicate isSink(DataFlow::Node sink) { sink = any(LogOutput logoutput).getAnInput() }
override predicate isSanitizer(DataFlow::Node node) {
exists(CallNode call |
node.asCfgNode() = call.getFunction().(AttrNode).getObject("replace") and
call.getArg(0).getNode().(StrConst).getText() in ["\r\n", "\n"]
)
}
}

57
python/ql/src/experimental/semmle/python/security/injection/NoSQLInjection.qll Normal file
View File

@@ -0,0 +1,57 @@
import python
import semmle.python.dataflow.new.DataFlow
import semmle.python.dataflow.new.DataFlow2
import semmle.python.dataflow.new.TaintTracking
import semmle.python.dataflow.new.TaintTracking2
import semmle.python.dataflow.new.RemoteFlowSources
import semmle.python.security.dataflow.ChainedConfigs12
import experimental.semmle.python.Concepts
import semmle.python.Concepts
/**
* A taint-tracking configuration for detecting string-to-dict conversions.
*/
class RFSToDictConfig extends TaintTracking::Configuration {
RFSToDictConfig() { this = "RFSToDictConfig" }
override predicate isSource(DataFlow::Node source) { source instanceof RemoteFlowSource }
override predicate isSink(DataFlow::Node sink) {
exists(Decoding decoding | decoding.getFormat() = "JSON" and sink = decoding.getOutput())
}
override predicate isSanitizer(DataFlow::Node sanitizer) {
sanitizer = any(NoSQLSanitizer noSQLSanitizer).getAnInput()
}
}
/**
* A taint-tracking configuration for detecting NoSQL injections (previously converted to a dict).
*/
class FromDataDictToSink extends TaintTracking2::Configuration {
FromDataDictToSink() { this = "FromDataDictToSink" }
override predicate isSource(DataFlow::Node source) {
exists(Decoding decoding | decoding.getFormat() = "JSON" and source = decoding.getOutput())
}
override predicate isSink(DataFlow::Node sink) { sink = any(NoSQLQuery noSQLQuery).getQuery() }
override predicate isSanitizer(DataFlow::Node sanitizer) {
sanitizer = any(NoSQLSanitizer noSQLSanitizer).getAnInput()
}
}
/**
* A predicate checking string-to-dict conversion and its arrival to a NoSQL injection sink.
*/
predicate noSQLInjectionFlow(CustomPathNode source, CustomPathNode sink) {
exists(
RFSToDictConfig config, DataFlow::PathNode mid1, DataFlow2::PathNode mid2,
FromDataDictToSink config2
|
config.hasFlowPath(source.asNode1(), mid1) and
config2.hasFlowPath(mid2, sink.asNode2()) and
mid1.getNode().asCfgNode() = mid2.getNode().asCfgNode()
)
}

53
python/ql/src/experimental/semmle/python/security/injection/RegexInjection.qll
View File

@@ -1,53 +0,0 @@
/**
* Provides a taint-tracking configuration for detecting regular expression injection
* vulnerabilities.
*/
import python
import experimental.semmle.python.Concepts
import semmle.python.dataflow.new.DataFlow
import semmle.python.dataflow.new.TaintTracking
import semmle.python.dataflow.new.RemoteFlowSources
/**
* A class to find methods executing regular expressions.
*
* See `RegexExecution`
*/
class RegexInjectionSink extends DataFlow::Node {
string regexModule;
Attribute regexMethod;
RegexInjectionSink() {
exists(RegexExecution reExec |
this = reExec.getRegexNode() and
regexModule = reExec.getRegexModule() and
regexMethod = reExec.(DataFlow::CallCfgNode).getFunction().asExpr().(Attribute)
)
}
/**
* Gets the argument containing the executed expression.
*/
string getRegexModule() { result = regexModule }
/**
* Gets the method used to execute the regular expression.
*/
Attribute getRegexMethod() { result = regexMethod }
}
/**
* A taint-tracking configuration for detecting regular expression injections.
*/
class RegexInjectionFlowConfig extends TaintTracking::Configuration {
RegexInjectionFlowConfig() { this = "RegexInjectionFlowConfig" }
override predicate isSource(DataFlow::Node source) { source instanceof RemoteFlowSource }
override predicate isSink(DataFlow::Node sink) { sink instanceof RegexInjectionSink }
override predicate isSanitizer(DataFlow::Node sanitizer) {
sanitizer = any(RegexEscape reEscape).getRegexNode()
}
}

115
python/ql/src/experimental/semmle/python/security/injection/Xpath.qll
View File

@@ -1,115 +0,0 @@
/**
* Provides class and predicates to track external data that
* may represent malicious xpath query objects.
*
* This module is intended to be imported into a taint-tracking query
* to extend `TaintKind` and `TaintSink`.
*/
import python
import semmle.python.dataflow.TaintTracking
import semmle.python.web.HttpRequest
/** Models Xpath Injection related classes and functions */
module XpathInjection {
/** Returns a class value which refers to `lxml.etree` */
Value etree() { result = Value::named("lxml.etree") }
/** Returns a class value which refers to `lxml.etree` */
Value libxml2parseFile() { result = Value::named("libxml2.parseFile") }
/** A generic taint sink that is vulnerable to Xpath injection. */
abstract class XpathInjectionSink extends TaintSink { }
/**
* A Sink representing an argument to the `etree.XPath` call.
*
* from lxml import etree
* root = etree.XML("<xmlContent>")
* find_text = etree.XPath("`sink`")
*/
private class EtreeXpathArgument extends XpathInjectionSink {
override string toString() { result = "lxml.etree.XPath" }
EtreeXpathArgument() {
exists(CallNode call | call.getFunction().(AttrNode).getObject("XPath").pointsTo(etree()) |
call.getArg(0) = this
)
}
override predicate sinks(TaintKind kind) { kind instanceof ExternalStringKind }
}
/**
* A Sink representing an argument to the `etree.EtXpath` call.
*
* from lxml import etree
* root = etree.XML("<xmlContent>")
* find_text = etree.EtXPath("`sink`")
*/
private class EtreeETXpathArgument extends XpathInjectionSink {
override string toString() { result = "lxml.etree.ETXpath" }
EtreeETXpathArgument() {
exists(CallNode call | call.getFunction().(AttrNode).getObject("ETXPath").pointsTo(etree()) |
call.getArg(0) = this
)
}
override predicate sinks(TaintKind kind) { kind instanceof ExternalStringKind }
}
/**
* A Sink representing an argument to the `xpath` call to a parsed xml document.
*
* from lxml import etree
* from io import StringIO
* f = StringIO('<foo><bar></bar></foo>')
* tree = etree.parse(f)
* r = tree.xpath('`sink`')
*/
private class ParseXpathArgument extends XpathInjectionSink {
override string toString() { result = "lxml.etree.parse.xpath" }
ParseXpathArgument() {
exists(
CallNode parseCall, CallNode xpathCall, ControlFlowNode obj, Variable var, AssignStmt assign
|
parseCall.getFunction().(AttrNode).getObject("parse").pointsTo(etree()) and
assign.getValue().(Call).getAFlowNode() = parseCall and
xpathCall.getFunction().(AttrNode).getObject("xpath") = obj and
var.getAUse() = obj and
assign.getATarget() = var.getAStore() and
xpathCall.getArg(0) = this
)
}
override predicate sinks(TaintKind kind) { kind instanceof ExternalStringKind }
}
/**
* A Sink representing an argument to the `xpathEval` call to a parsed libxml2 document.
*
* import libxml2
* tree = libxml2.parseFile("file.xml")
* r = tree.xpathEval('`sink`')
*/
private class ParseFileXpathEvalArgument extends XpathInjectionSink {
override string toString() { result = "libxml2.parseFile.xpathEval" }
ParseFileXpathEvalArgument() {
exists(
CallNode parseCall, CallNode xpathCall, ControlFlowNode obj, Variable var, AssignStmt assign
|
parseCall.getFunction().(AttrNode).pointsTo(libxml2parseFile()) and
assign.getValue().(Call).getAFlowNode() = parseCall and
xpathCall.getFunction().(AttrNode).getObject("xpathEval") = obj and
var.getAUse() = obj and
assign.getATarget() = var.getAStore() and
xpathCall.getArg(0) = this
)
}
override predicate sinks(TaintKind kind) { kind instanceof ExternalStringKind }
}
}

4
python/ql/src/external/DefectFilter.qll vendored
View File

@@ -8,7 +8,7 @@ import semmle.python.Files
* column `startcol` of line `startline` to column `endcol` of line `endline`
* in file `filepath`.
*
* For more information, see [Locations](https://help.semmle.com/QL/learn-ql/ql/locations.html).
* For more information, see [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/).
*/
external predicate defectResults(
int id, string queryPath, string filepath, int startline, int startcol, int endline, int endcol,
@@ -54,7 +54,7 @@ class DefectResult extends int {
* The location spans column `startcolumn` of line `startline` to
* column `endcolumn` of line `endline` in file `filepath`.
* For more information, see
* [Locations](https://help.semmle.com/QL/learn-ql/ql/locations.html).
* [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/).
*/
predicate hasLocationInfo(
string filepath, int startline, int startcolumn, int endline, int endcolumn

2
python/ql/src/external/Thrift.qll vendored
View File

@@ -38,7 +38,7 @@ class ThriftElement extends ExternalData {
* The location spans column `startcolumn` of line `startline` to
* column `endcolumn` of line `endline` in file `filepath`.
* For more information, see
* [Locations](https://help.semmle.com/QL/learn-ql/ql/locations.html).
* [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/).
*/
predicate hasLocationInfo(
string filepath, int startline, int startcolumn, int endline, int endcolumn

18
python/ql/src/meta/alerts/RemoteFlowSources.ql Normal file
View File

@@ -0,0 +1,18 @@
/**
* @name Remote flow sources
* @description Sources of remote user input.
* @kind problem
* @problem.severity recommendation
* @id py/meta/alerts/remote-flow-sources
* @tags meta
* @precision very-low
*/
private import python
private import semmle.python.dataflow.new.DataFlow
private import semmle.python.dataflow.new.RemoteFlowSources
private import meta.MetaMetrics
from RemoteFlowSource source
where not source.getLocation().getFile() instanceof IgnoredFile
select source, "RemoteFlowSource: " + source.getSourceType()

55
python/ql/src/meta/alerts/RemoteFlowSourcesReach.ql Normal file
View File

@@ -0,0 +1,55 @@
/**
* @name Remote flow sources reach
* @description Nodes that can be reached with taint tracking from sources of
* remote user input.
* @kind problem
* @problem.severity recommendation
* @id py/meta/alerts/remote-flow-sources-reach
* @tags meta
* @precision very-low
*/
private import python
private import semmle.python.dataflow.new.DataFlow
private import semmle.python.dataflow.new.TaintTracking
private import semmle.python.dataflow.new.RemoteFlowSources
private import meta.MetaMetrics
private import semmle.python.dataflow.new.internal.PrintNode
class RemoteFlowSourceReach extends TaintTracking::Configuration {
RemoteFlowSourceReach() { this = "RemoteFlowSourceReach" }
override predicate isSource(DataFlow::Node node) {
node instanceof RemoteFlowSource and
not node.getLocation().getFile() instanceof IgnoredFile
}
override predicate isSink(DataFlow::Node node) {
not node.getLocation().getFile() instanceof IgnoredFile and
(
node instanceof RemoteFlowSource
or
this.isAdditionalFlowStep(_, node)
) and
// In september 2021 we changed how we do taint-propagation for method calls (mostly
// relating to modeled frameworks/libraries). We used to do `obj -> obj.meth` and
// `obj.meth -> obj.meth()` in two separate steps, and now do them in one
// `obj -> obj.meth()`. To be able to compare the overall reach between these two
// version, we don't want this query to alert us to the fact that we no longer taint
// the node in the middle (since that is just noise).
// see https://github.com/github/codeql/pull/6349
//
// We should be able to remove the following few lines of code once we don't care to
// compare with the old (before September 2021) way of doing taint-propagation for
// method calls.
not exists(DataFlow::MethodCallNode c |
node = c.getFunction() and
this.isAdditionalFlowStep(c.getObject(), node) and
this.isAdditionalFlowStep(node, c)
)
}
}
from RemoteFlowSourceReach cfg, DataFlow::Node reachable
where cfg.hasFlow(_, reachable)
select reachable, prettyNode(reachable)

43
python/ql/src/python.qll
View File

@@ -1,43 +0,0 @@
import Customizations
import semmle.python.Files
import semmle.python.Operations
import semmle.python.Variables
import semmle.python.AstGenerated
import semmle.python.AstExtended
import semmle.python.Function
import semmle.python.Module
import semmle.python.Class
import semmle.python.Import
import semmle.python.Stmts
import semmle.python.Exprs
import semmle.python.Keywords
import semmle.python.Comprehensions
import semmle.python.Flow
import semmle.python.Metrics
import semmle.python.Constants
import semmle.python.Scope
import semmle.python.Comment
import semmle.python.GuardedControlFlow
import semmle.python.types.ImportTime
import semmle.python.types.Object
import semmle.python.types.ClassObject
import semmle.python.types.FunctionObject
import semmle.python.types.ModuleObject
import semmle.python.types.Version
import semmle.python.types.Descriptors
import semmle.python.protocols
import semmle.python.SSA
import semmle.python.SelfAttribute
import semmle.python.types.Properties
import semmle.python.xml.XML
import semmle.python.essa.Essa
import semmle.python.pointsto.Base
import semmle.python.pointsto.Context
import semmle.python.pointsto.CallGraph
import semmle.python.objects.ObjectAPI
import semmle.python.Unit
import site
// Removing this import perturbs the compilation process enough that the points-to analysis gets
// compiled -- and cached -- differently depending on whether the data flow library is imported. By
// importing it privately here, we ensure that the points-to analysis is compiled the same way.
private import semmle.python.dataflow.new.DataFlow

4
python/ql/src/qlpack.lock.yml Normal file
View File

@@ -0,0 +1,4 @@
---
dependencies: {}
compiled: false
lockVersion: 1.0.0

9
python/ql/src/qlpack.yml
View File

@@ -1,5 +1,8 @@
name: codeql-python
version: 0.0.0
dbscheme: semmlecode.python.dbscheme
name: codeql/python-queries
version: 0.0.2
dependencies:
codeql/python-all: "*"
codeql/suite-helpers: "*"
suites: codeql-suites
extractor: python
defaultSuiteFile: codeql-suites/python-code-scanning.qls

3
python/ql/src/semmle/crypto/Crypto.qll
View File

@@ -1,3 +0,0 @@
/** DEPRECATED: Use `semmle.python.concepts.CryptoAlgorithms` instead. */
import semmle.python.concepts.CryptoAlgorithms

3
python/ql/src/semmle/dataflow/SSA.qll
View File

@@ -1,3 +0,0 @@
/** For backward compatibility */
import semmle.python.essa.Essa

3
python/ql/src/semmle/files/FileSystem.qll
View File

@@ -1,3 +0,0 @@
/** Provides classes for working with files and folders. */
import semmle.python.Files

605
python/ql/src/semmle/python/ApiGraphs.qll
View File

@@ -1,605 +0,0 @@
/**
* Provides an implementation of _API graphs_, which are an abstract representation of the API
* surface used and/or defined by a code base.
*
* The nodes of the API graph represent definitions and uses of API components. The edges are
* directed and labeled; they specify how the components represented by nodes relate to each other.
*/
private import python
import semmle.python.dataflow.new.DataFlow
/**
* Provides classes and predicates for working with APIs used in a database.
*/
module API {
/**
* An abstract representation of a definition or use of an API component such as a function
* exported by a Python package, or its result.
*/
class Node extends Impl::TApiNode {
/**
* Gets a data-flow node corresponding to a use of the API component represented by this node.
*
* For example, `import re; re.escape` is a use of the `escape` function from the
* `re` module, and `import re; re.escape("hello")` is a use of the return of that function.
*
* This includes indirect uses found via data flow, meaning that in
* ```python
* def f(x):
* pass
*
* f(obj.foo)
* ```
* both `obj.foo` and `x` are uses of the `foo` member from `obj`.
*/
DataFlow::Node getAUse() {
exists(DataFlow::LocalSourceNode src | Impl::use(this, src) |
Impl::trackUseNode(src).flowsTo(result)
)
}
/**
* Gets an immediate use of the API component represented by this node.
*
* For example, `import re; re.escape` is a an immediate use of the `escape` member
* from the `re` module.
*
* Unlike `getAUse()`, this predicate only gets the immediate references, not the indirect uses
* found via data flow. This means that in `x = re.escape` only `re.escape` is a reference
* to the `escape` member of `re`, neither `x` nor any node that `x` flows to is a reference to
* this API component.
*/
DataFlow::LocalSourceNode getAnImmediateUse() { Impl::use(this, result) }
/**
* Gets a call to the function represented by this API component.
*/
DataFlow::CallCfgNode getACall() { result = getReturn().getAnImmediateUse() }
/**
* Gets a node representing member `m` of this API component.
*
* For example, a member can be:
*
* - A submodule of a module
* - An attribute of an object
*/
bindingset[m]
bindingset[result]
Node getMember(string m) { result = getASuccessor(Label::member(m)) }
/**
* Gets a node representing a member of this API component where the name of the member is
* not known statically.
*/
Node getUnknownMember() { result = getASuccessor(Label::unknownMember()) }
/**
* Gets a node representing a member of this API component where the name of the member may
* or may not be known statically.
*/
Node getAMember() {
result = getASuccessor(Label::member(_)) or
result = getUnknownMember()
}
/**
* Gets a node representing the result of the function represented by this node.
*
* This predicate may have multiple results when there are multiple invocations of this API component.
* Consider using `getACall()` if there is a need to distinguish between individual calls.
*/
Node getReturn() { result = getASuccessor(Label::return()) }
/**
* Gets a node representing a subclass of the class represented by this node.
*/
Node getASubclass() { result = getASuccessor(Label::subclass()) }
/**
* Gets a node representing the result from awaiting this node.
*/
Node getAwaited() { result = getASuccessor(Label::await()) }
/**
* Gets a string representation of the lexicographically least among all shortest access paths
* from the root to this node.
*/
string getPath() { result = min(string p | p = getAPath(Impl::distanceFromRoot(this)) | p) }
/**
* Gets a node such that there is an edge in the API graph between this node and the other
* one, and that edge is labeled with `lbl`.
*/
Node getASuccessor(string lbl) { Impl::edge(this, lbl, result) }
/**
* Gets a node such that there is an edge in the API graph between that other node and
* this one, and that edge is labeled with `lbl`
*/
Node getAPredecessor(string lbl) { this = result.getASuccessor(lbl) }
/**
* Gets a node such that there is an edge in the API graph between this node and the other
* one.
*/
Node getAPredecessor() { result = getAPredecessor(_) }
/**
* Gets a node such that there is an edge in the API graph between that other node and
* this one.
*/
Node getASuccessor() { result = getASuccessor(_) }
/**
* Gets the data-flow node that gives rise to this node, if any.
*/
DataFlow::Node getInducingNode() { this = Impl::MkUse(result) }
/**
* Holds if this element is at the specified location.
* The location spans column `startcolumn` of line `startline` to
* column `endcolumn` of line `endline` in file `filepath`.
* For more information, see
* [Locations](https://help.semmle.com/QL/learn-ql/locations.html).
*/
predicate hasLocationInfo(
string filepath, int startline, int startcolumn, int endline, int endcolumn
) {
getInducingNode().hasLocationInfo(filepath, startline, startcolumn, endline, endcolumn)
or
// For nodes that do not have a meaningful location, `path` is the empty string and all other
// parameters are zero.
not exists(getInducingNode()) and
filepath = "" and
startline = 0 and
startcolumn = 0 and
endline = 0 and
endcolumn = 0
}
/**
* Gets a textual representation of this element.
*/
abstract string toString();
/**
* Gets a path of the given `length` from the root to this node.
*/
private string getAPath(int length) {
this instanceof Impl::MkRoot and
length = 0 and
result = ""
or
exists(Node pred, string lbl, string predpath |
Impl::edge(pred, lbl, this) and
lbl != "" and
predpath = pred.getAPath(length - 1) and
exists(string dot | if length = 1 then dot = "" else dot = "." |
result = predpath + dot + lbl and
// avoid producing strings longer than 1MB
result.length() < 1000 * 1000
)
) and
length in [1 .. Impl::distanceFromRoot(this)]
}
/** Gets the shortest distance from the root to this node in the API graph. */
int getDepth() { result = Impl::distanceFromRoot(this) }
}
/** The root node of an API graph. */
class Root extends Node, Impl::MkRoot {
override string toString() { result = "root" }
}
/** A node corresponding to the use of an API component. */
class Use extends Node, Impl::TUse {
override string toString() {
exists(string type |
this = Impl::MkUse(_) and type = "Use "
or
this = Impl::MkModuleImport(_) and type = "ModuleImport "
|
result = type + getPath()
or
not exists(this.getPath()) and result = type + "with no path"
)
}
}
/** Gets the root node. */
Root root() { any() }
/**
* Gets a node corresponding to an import of module `m`.
*
* Note: You should only use this predicate for top level modules. If you want nodes corresponding to a submodule,
* you should use `.getMember` on the parent module. For example, for nodes corresponding to the module `foo.bar`,
* use `moduleImport("foo").getMember("bar")`.
*/
Node moduleImport(string m) { result = Impl::MkModuleImport(m) }
/** Gets a node corresponding to the built-in with the given name, if any. */
Node builtin(string n) { result = moduleImport("builtins").getMember(n) }
/**
* Provides the actual implementation of API graphs, cached for performance.
*
* Ideally, we'd like nodes to correspond to (global) access paths, with edge labels
* corresponding to extending the access path by one element. We also want to be able to map
* nodes to their definitions and uses in the data-flow graph, and this should happen modulo
* (inter-procedural) data flow.
*
* This, however, is not easy to implement, since access paths can have unbounded length
* and we need some way of recognizing cycles to avoid non-termination. Unfortunately, expressing
* a condition like "this node hasn't been involved in constructing any predecessor of
* this node in the API graph" without negative recursion is tricky.
*
* So instead most nodes are directly associated with a data-flow node, representing
* either a use or a definition of an API component. This ensures that we only have a finite
* number of nodes. However, we can now have multiple nodes with the same access
* path, which are essentially indistinguishable for a client of the API.
*
* On the other hand, a single node can have multiple access paths (which is, of
* course, unavoidable). We pick as canonical the alphabetically least access path with
* shortest length.
*/
cached
private module Impl {
/*
* Modeling imports is slightly tricky because of the way we handle dotted name imports in our
* libraries. In dotted imports such as
*
* ```python
* import foo.bar.baz as fbb
* from foo.bar.baz import quux as fbbq
* ```
*
* the dotted name is simply represented as a string. We would like `fbb.quux` and `fbbq` to
* be represented as API graph nodes with the following path:
*
* ```ql
* moduleImport("foo").getMember("bar").getMember("baz").getMember("quux")
* ```
*
* To do this, we produce an API graph node for each dotted name prefix we find in the set of
* imports. Thus, for the above two imports, we would get nodes for
*
* ```python
* foo
* foo.bar
* foo.bar.baz
* ```
*
* Only the first of these can act as the beginning of a path (and become a
* `moduleImport`-labeled edge from the global root node).
*
* (Using prefixes rather than simply `foo`, `bar`, and `baz` is important. We don't want
* potential crosstalk between `foo.bar.baz` and `ham.bar.eggs`.)
*
* We then add `getMember` edges between these prefixes: `foo` steps to `foo.bar` via an edge
* labeled `getMember("bar")` and so on.
*
* When we then see `import foo.bar.baz as fbb`, the data-flow node `fbb` gets marked as a use
* of the API graph node corresponding to the prefix `foo.bar.baz`. Because of the edges leading to
* this node, it is reachable via `moduleImport("foo").getMember("bar").getMember("baz")` and
* thus `fbb.quux` is reachable via the path mentioned above.
*
* When we see `from foo.bar.baz import quux as fbbq` a similar thing happens. First, `foo.bar.baz`
* is seen as a use of the API graph node as before. Then `import quux as fbbq` is seen as
* a member lookup of `quux` on the API graph node for `foo.bar.baz`, and then finally the
* data-flow node `fbbq` is marked as a use of the same path mentioned above.
*
* Finally, in a non-aliased import such as
*
* ```python
* import foo.bar.baz
* ```
*
* we only consider this as a definition of the name `foo` (thus making it a use of the corresponding
* API graph node for the prefix `foo`), in accordance with the usual semantics of Python.
*/
cached
newtype TApiNode =
/** The root of the API graph. */
MkRoot() or
/** An abstract representative for imports of the module called `name`. */
MkModuleImport(string name) {
// Ignore the following module name for Python 2, as we alias `__builtin__` to `builtins` elsewhere
(name != "__builtin__" or major_version() = 3) and
(
imports(_, name)
or
// When we `import foo.bar.baz` we want to create API graph nodes also for the prefixes
// `foo` and `foo.bar`:
name = any(ImportExpr e | not e.isRelative()).getAnImportedModuleName()
)
or
// The `builtins` module should always be implicitly available
name = "builtins"
} or
/** A use of an API member at the node `nd`. */
MkUse(DataFlow::Node nd) { use(_, _, nd) }
class TUse = MkModuleImport or MkUse;
/**
* Holds if the dotted module name `sub` refers to the `member` member of `base`.
*
* For instance, `prefix_member("foo.bar", "baz", "foo.bar.baz")` would hold.
*/
private predicate prefix_member(TApiNode base, string member, TApiNode sub) {
exists(string sub_str, string regexp |
regexp = "(.+)[.]([^.]+)" and
base = MkModuleImport(sub_str.regexpCapture(regexp, 1)) and
member = sub_str.regexpCapture(regexp, 2) and
sub = MkModuleImport(sub_str)
)
}
/**
* Holds if `imp` is a data-flow node inside an import statement that refers to a module by the
* name `name`.
*
* Ignores relative imports, such as `from ..foo.bar import baz`.
*/
private predicate imports(DataFlow::Node imp, string name) {
exists(ImportExprNode iexpr |
imp.asCfgNode() = iexpr and
not iexpr.getNode().isRelative() and
name = iexpr.getNode().getImportedModuleName()
)
}
/** Gets the name of a known built-in. */
private string getBuiltInName() {
// These lists were created by inspecting the `builtins` and `__builtin__` modules in
// Python 3 and 2 respectively, using the `dir` built-in.
// Built-in functions and exceptions shared between Python 2 and 3
result in [
"abs", "all", "any", "bin", "bool", "bytearray", "callable", "chr", "classmethod",
"compile", "complex", "delattr", "dict", "dir", "divmod", "enumerate", "eval", "filter",
"float", "format", "frozenset", "getattr", "globals", "hasattr", "hash", "help", "hex",
"id", "input", "int", "isinstance", "issubclass", "iter", "len", "list", "locals", "map",
"max", "memoryview", "min", "next", "object", "oct", "open", "ord", "pow", "print",
"property", "range", "repr", "reversed", "round", "set", "setattr", "slice", "sorted",
"staticmethod", "str", "sum", "super", "tuple", "type", "vars", "zip", "__import__",
// Exceptions
"ArithmeticError", "AssertionError", "AttributeError", "BaseException", "BufferError",
"BytesWarning", "DeprecationWarning", "EOFError", "EnvironmentError", "Exception",
"FloatingPointError", "FutureWarning", "GeneratorExit", "IOError", "ImportError",
"ImportWarning", "IndentationError", "IndexError", "KeyError", "KeyboardInterrupt",
"LookupError", "MemoryError", "NameError", "NotImplemented", "NotImplementedError",
"OSError", "OverflowError", "PendingDeprecationWarning", "ReferenceError", "RuntimeError",
"RuntimeWarning", "StandardError", "StopIteration", "SyntaxError", "SyntaxWarning",
"SystemError", "SystemExit", "TabError", "TypeError", "UnboundLocalError",
"UnicodeDecodeError", "UnicodeEncodeError", "UnicodeError", "UnicodeTranslateError",
"UnicodeWarning", "UserWarning", "ValueError", "Warning", "ZeroDivisionError",
// Added for compatibility
"exec"
]
or
// Built-in constants shared between Python 2 and 3
result in ["False", "True", "None", "NotImplemented", "Ellipsis", "__debug__"]
or
// Python 3 only
result in [
"ascii", "breakpoint", "bytes", "exec",
// Exceptions
"BlockingIOError", "BrokenPipeError", "ChildProcessError", "ConnectionAbortedError",
"ConnectionError", "ConnectionRefusedError", "ConnectionResetError", "FileExistsError",
"FileNotFoundError", "InterruptedError", "IsADirectoryError", "ModuleNotFoundError",
"NotADirectoryError", "PermissionError", "ProcessLookupError", "RecursionError",
"ResourceWarning", "StopAsyncIteration", "TimeoutError"
]
or
// Python 2 only
result in [
"basestring", "cmp", "execfile", "file", "long", "raw_input", "reduce", "reload",
"unichr", "unicode", "xrange"
]
}
/**
* Gets a data flow node that is likely to refer to a built-in with the name `name`.
*
* Currently this is an over-approximation, and may not account for things like overwriting a
* built-in with a different value.
*/
private DataFlow::Node likely_builtin(string name) {
exists(Module m |
result.asCfgNode() =
any(NameNode n |
possible_builtin_accessed_in_module(n, name, m) and
not possible_builtin_defined_in_module(name, m)
)
)
}
/**
* Holds if a global variable called `name` (which is also the name of a built-in) is assigned
* a value in the module `m`.
*/
private predicate possible_builtin_defined_in_module(string name, Module m) {
exists(NameNode n |
not exists(LocalVariable v | n.defines(v)) and
n.isStore() and
name = n.getId() and
name = getBuiltInName() and
m = n.getEnclosingModule()
)
}
/**
* Holds if `n` is an access of a global variable called `name` (which is also the name of a
* built-in) inside the module `m`.
*/
private predicate possible_builtin_accessed_in_module(NameNode n, string name, Module m) {
n.isGlobal() and
n.isLoad() and
name = n.getId() and
name = getBuiltInName() and
m = n.getEnclosingModule()
}
/**
* Holds if `ref` is a use of a node that should have an incoming edge from `base` labeled
* `lbl` in the API graph.
*/
cached
predicate use(TApiNode base, string lbl, DataFlow::Node ref) {
exists(DataFlow::LocalSourceNode src, DataFlow::LocalSourceNode pred |
// First, we find a predecessor of the node `ref` that we want to determine. The predecessor
// is any node that is a type-tracked use of a data flow node (`src`), which is itself a
// reference to the API node `base`. Thus, `pred` and `src` both represent uses of `base`.
//
// Once we have identified the predecessor, we define its relation to the successor `ref` as
// well as the label on the edge from `pred` to `ref`. This label describes the nature of
// the relationship between `pred` and `ref`.
use(base, src) and pred = trackUseNode(src)
|
// Referring to an attribute on a node that is a use of `base`:
lbl = Label::memberFromRef(ref) and
ref = pred.getAnAttributeReference()
or
// Calling a node that is a use of `base`
lbl = Label::return() and
ref = pred.getACall()
or
// Subclassing a node
lbl = Label::subclass() and
exists(DataFlow::Node superclass | pred.flowsTo(superclass) |
ref.asExpr().(ClassExpr).getABase() = superclass.asExpr()
)
or
// awaiting
exists(Await await, DataFlow::Node awaitedValue |
lbl = Label::await() and
ref.asExpr() = await and
await.getValue() = awaitedValue.asExpr() and
pred.flowsTo(awaitedValue)
)
)
or
// Built-ins, treated as members of the module `builtins`
base = MkModuleImport("builtins") and
lbl = Label::member(any(string name | ref = likely_builtin(name)))
}
/**
* Holds if `ref` is a use of node `nd`.
*/
cached
predicate use(TApiNode nd, DataFlow::Node ref) {
exists(string name |
nd = MkModuleImport(name) and
imports(ref, name)
)
or
// Ensure the Python 2 `__builtin__` module gets the name of the Python 3 `builtins` module.
major_version() = 2 and
nd = MkModuleImport("builtins") and
imports(ref, "__builtin__")
or
nd = MkUse(ref)
}
/**
* Gets a data-flow node to which `src`, which is a use of an API-graph node, flows.
*
* The flow from `src` to that node may be inter-procedural.
*/
private DataFlow::TypeTrackingNode trackUseNode(
DataFlow::LocalSourceNode src, DataFlow::TypeTracker t
) {
t.start() and
use(_, src) and
result = src
or
exists(DataFlow::TypeTracker t2 | result = trackUseNode(src, t2).track(t2, t))
}
/**
* Gets a data-flow node to which `src`, which is a use of an API-graph node, flows.
*
* The flow from `src` to that node may be inter-procedural.
*/
cached
DataFlow::LocalSourceNode trackUseNode(DataFlow::LocalSourceNode src) {
result = trackUseNode(src, DataFlow::TypeTracker::end()) and
not result instanceof DataFlow::ModuleVariableNode
}
/**
* Holds if there is an edge from `pred` to `succ` in the API graph that is labeled with `lbl`.
*/
cached
predicate edge(TApiNode pred, string lbl, TApiNode succ) {
/* There's an edge from the root node for each imported module. */
exists(string m |
pred = MkRoot() and
lbl = Label::mod(m)
|
succ = MkModuleImport(m) and
// Only allow undotted names to count as base modules.
not m.matches("%.%")
)
or
/* Step from the dotted module name `foo.bar` to `foo.bar.baz` along an edge labeled `baz` */
exists(string member |
prefix_member(pred, member, succ) and
lbl = Label::member(member)
)
or
/* Every node that is a use of an API component is itself added to the API graph. */
exists(DataFlow::LocalSourceNode ref |
use(pred, lbl, ref) and
succ = MkUse(ref)
)
}
/**
* Holds if there is an edge from `pred` to `succ` in the API graph.
*/
private predicate edge(TApiNode pred, TApiNode succ) { edge(pred, _, succ) }
/** Gets the shortest distance from the root to `nd` in the API graph. */
cached
int distanceFromRoot(TApiNode nd) = shortestDistances(MkRoot/0, edge/2)(_, nd, result)
}
}
private module Label {
/** Gets the edge label for the module `m`. */
bindingset[m]
bindingset[result]
string mod(string m) { result = "moduleImport(\"" + m + "\")" }
/** Gets the `member` edge label for member `m`. */
bindingset[m]
bindingset[result]
string member(string m) { result = "getMember(\"" + m + "\")" }
/** Gets the `member` edge label for the unknown member. */
string unknownMember() { result = "getUnknownMember()" }
/** Gets the `member` edge label for the given attribute reference. */
string memberFromRef(DataFlow::AttrRef pr) {
result = member(pr.getAttributeName())
or
not exists(pr.getAttributeName()) and
result = unknownMember()
}
/** Gets the `return` edge label. */
string return() { result = "getReturn()" }
/** Gets the `subclass` edge label. */
string subclass() { result = "getASubclass()" }
/** Gets the `await` edge label. */
string await() { result = "getAwaited()" }
}

170
python/ql/src/semmle/python/AstExtended.qll
View File

@@ -1,170 +0,0 @@
import python
/** Syntactic node (Class, Function, Module, Expr, Stmt or Comprehension) corresponding to a flow node */
abstract class AstNode extends AstNode_ {
/*
* Special comment for documentation generation.
* All subclasses of `AstNode` that represent concrete syntax should have
* a comment of the form:
*/
/* syntax: ... */
/** Gets the scope that this node occurs in */
abstract Scope getScope();
/**
* Gets a flow node corresponding directly to this node.
* NOTE: For some statements and other purely syntactic elements,
* there may not be a `ControlFlowNode`
*/
ControlFlowNode getAFlowNode() { py_flow_bb_node(result, this, _, _) }
/** Gets the location for this AST node */
Location getLocation() { none() }
/**
* Whether this syntactic element is artificial, that is it is generated
* by the compiler and is not present in the source
*/
predicate isArtificial() { none() }
/**
* Gets a child node of this node in the AST. This predicate exists to aid exploration of the AST
* and other experiments. The child-parent relation may not be meaningful.
* For a more meaningful relation in terms of dependency use
* Expr.getASubExpression(), Stmt.getASubStatement(), Stmt.getASubExpression() or
* Scope.getAStmt().
*/
abstract AstNode getAChildNode();
/**
* Gets the parent node of this node in the AST. This predicate exists to aid exploration of the AST
* and other experiments. The child-parent relation may not be meaningful.
* For a more meaningful relation in terms of dependency use
* Expr.getASubExpression(), Stmt.getASubStatement(), Stmt.getASubExpression() or
* Scope.getAStmt() applied to the parent.
*/
AstNode getParentNode() { result.getAChildNode() = this }
/** Whether this contains `inner` syntactically */
predicate contains(AstNode inner) { this.getAChildNode+() = inner }
/** Whether this contains `inner` syntactically and `inner` has the same scope as `this` */
predicate containsInScope(AstNode inner) {
this.contains(inner) and
this.getScope() = inner.getScope() and
not inner instanceof Scope
}
}
/* Parents */
/** Internal implementation class */
library class FunctionParent extends FunctionParent_ { }
/** Internal implementation class */
library class ArgumentsParent extends ArgumentsParent_ { }
/** Internal implementation class */
library class ExprListParent extends ExprListParent_ { }
/** Internal implementation class */
library class ExprContextParent extends ExprContextParent_ { }
/** Internal implementation class */
library class StmtListParent extends StmtListParent_ { }
/** Internal implementation class */
library class StrListParent extends StrListParent_ { }
/** Internal implementation class */
library class ExprParent extends ExprParent_ { }
library class DictItem extends DictItem_, AstNode {
override string toString() { result = DictItem_.super.toString() }
override AstNode getAChildNode() { none() }
override Scope getScope() { none() }
}
/** A comprehension part, the 'for a in seq' part of [ a * a for a in seq ] */
class Comprehension extends Comprehension_, AstNode {
/** Gets the scope of this comprehension */
override Scope getScope() {
/* Comprehensions exists only in Python 2 list comprehensions, so their scope is that of the list comp. */
exists(ListComp l | this = l.getAGenerator() | result = l.getScope())
}
override string toString() { result = "Comprehension" }
override Location getLocation() { result = Comprehension_.super.getLocation() }
override AstNode getAChildNode() { result = this.getASubExpression() }
Expr getASubExpression() {
result = this.getIter() or
result = this.getAnIf() or
result = this.getTarget()
}
}
class BytesOrStr extends BytesOrStr_ { }
/**
* Part of a string literal formed by implicit concatenation.
* For example the string literal "abc" expressed in the source as `"a" "b" "c"`
* would be composed of three `StringPart`s.
*/
class StringPart extends StringPart_, AstNode {
override Scope getScope() {
exists(Bytes b | this = b.getAnImplicitlyConcatenatedPart() | result = b.getScope())
or
exists(Unicode u | this = u.getAnImplicitlyConcatenatedPart() | result = u.getScope())
}
override AstNode getAChildNode() { none() }
override string toString() { result = StringPart_.super.toString() }
override Location getLocation() { result = StringPart_.super.getLocation() }
}
class StringPartList extends StringPartList_ { }
/* **** Lists ***/
/** A parameter list */
class ParameterList extends @py_parameter_list {
Function getParent() { py_parameter_lists(this, result) }
/** Gets a parameter */
Parameter getAnItem() {
/* Item can be a Name or a Tuple, both of which are expressions */
py_exprs(result, _, this, _)
}
/** Gets the nth parameter */
Parameter getItem(int index) {
/* Item can be a Name or a Tuple, both of which are expressions */
py_exprs(result, _, this, index)
}
string toString() { result = "ParameterList" }
}
/** A list of Comprehensions (for generating parts of a set, list or dictionary comprehension) */
class ComprehensionList extends ComprehensionList_ { }
/** A list of expressions */
class ExprList extends ExprList_ {
/* syntax: Expr, ... */
}
library class DictItemList extends DictItemList_ { }
library class DictItemListParent extends DictItemListParent_ { }
/** A list of strings (the primitive type string not Bytes or Unicode) */
class StringList extends StringList_ { }
/** A list of aliases in an import statement */
class AliasList extends AliasList_ { }

1646
python/ql/src/semmle/python/AstGenerated.qll
View File

File diff suppressed because it is too large Load Diff

175
python/ql/src/semmle/python/Class.qll
View File

@@ -1,175 +0,0 @@
/**
* Provides classes representing Python classes.
*/
import python
/**
* An (artificial) expression corresponding to a class definition.
* It is recommended to use `ClassDef` instead.
*/
class ClassExpr extends ClassExpr_ {
/** Gets the metaclass expression */
Expr getMetaClass() {
if major_version() = 3
then
exists(Keyword metacls |
this.getAKeyword() = metacls and
metacls.getArg() = "metaclass" and
result = metacls.getValue()
)
else
exists(Assign a |
a = this.getInnerScope().getAStmt() and
a.getATarget().(Name).getId() = "__metaclass__" and
result = a.getValue()
)
}
/** Gets the nth keyword argument of this class definition. */
override DictUnpackingOrKeyword getKeyword(int index) {
result = this.getKeywords().getItem(index)
}
/** Gets a keyword argument of this class definition. */
override DictUnpackingOrKeyword getAKeyword() { result = this.getKeywords().getAnItem() }
override Expr getASubExpression() {
result = this.getABase() or
result = this.getAKeyword().getValue() or
result = this.getKwargs() or
result = this.getStarargs()
}
/** Gets a call corresponding to a decorator of this class definition. */
Call getADecoratorCall() {
result.getArg(0) = this or
result.getArg(0) = this.getADecoratorCall()
}
/** Gets a decorator of this function expression */
Expr getADecorator() { result = this.getADecoratorCall().getFunc() }
override AstNode getAChildNode() {
result = this.getASubExpression()
or
result = this.getInnerScope()
}
/** Gets a tuple (*) argument of this class definition. */
Expr getStarargs() { result = this.getABase().(Starred).getValue() }
/** Gets a dictionary (**) argument of this class definition. */
Expr getKwargs() { result = this.getAKeyword().(DictUnpacking).getValue() }
}
/** A class statement. Note that ClassDef extends Assign as a class definition binds the newly created class */
class ClassDef extends Assign {
/* syntax: class name(...): ... */
ClassDef() {
/* This is an artificial assignment the rhs of which is a (possibly decorated) ClassExpr */
exists(ClassExpr c | this.getValue() = c or this.getValue() = c.getADecoratorCall())
}
override string toString() { result = "ClassDef" }
/** Gets the class for this statement */
Class getDefinedClass() {
exists(ClassExpr c | this.getValue() = c or this.getValue() = c.getADecoratorCall() |
result = c.getInnerScope()
)
}
override Stmt getLastStatement() { result = this.getDefinedClass().getLastStatement() }
}
/** The scope of a class. This is the scope of all the statements within the class definition */
class Class extends Class_, Scope, AstNode {
/**
* Use getADecorator() instead of getDefinition().getADecorator()
* Use getMetaClass() instead of getDefinition().getMetaClass()
*/
deprecated ClassExpr getDefinition() { result = this.getParent() }
/** Gets a defined init method of this class */
Function getInitMethod() { result.getScope() = this and result.isInitMethod() }
/** Gets a method defined in this class */
Function getAMethod() { result.getScope() = this }
override Location getLocation() { py_scope_location(result, this) }
/** Gets the scope (module, class or function) in which this class is defined */
override Scope getEnclosingScope() { result = this.getParent().getScope() }
/** Use getEnclosingScope() instead */
override Scope getScope() { result = this.getParent().getScope() }
override string toString() { result = "Class " + this.getName() }
/** Gets the statements forming the body of this class */
override StmtList getBody() { result = Class_.super.getBody() }
/** Gets the nth statement in the class */
override Stmt getStmt(int index) { result = Class_.super.getStmt(index) }
/** Gets a statement in the class */
override Stmt getAStmt() { result = Class_.super.getAStmt() }
/** Gets the name used to define this class */
override string getName() { result = Class_.super.getName() }
/** Holds if this expression may have a side effect (as determined purely from its syntax). */
predicate hasSideEffects() { any() }
/** Holds if this is probably a mixin (has 'mixin' or similar in name or docstring) */
predicate isProbableMixin() {
(
this.getName().toLowerCase().matches("%mixin%")
or
this.getDocString().getText().toLowerCase().matches("%mixin%")
or
this.getDocString().getText().toLowerCase().matches("%mix-in%")
)
}
override AstNode getAChildNode() { result = this.getAStmt() }
/** Gets a decorator of this class. */
Expr getADecorator() { result = this.getParent().getADecorator() }
/** Gets the metaclass expression */
Expr getMetaClass() { result = this.getParent().getMetaClass() }
/** Gets the ClassObject corresponding to this class */
ClassObject getClassObject() { result.getOrigin() = this.getParent() }
/** Gets the nth base of this class definition. */
Expr getBase(int index) { result = this.getParent().getBase(index) }
/** Gets a base of this class definition. */
Expr getABase() { result = this.getParent().getABase() }
/** Gets the metrics for this class */
ClassMetrics getMetrics() { result = this }
/**
* Gets the qualified name for this class.
* Should return the same name as the `__qualname__` attribute on classes in Python 3.
*/
string getQualifiedName() {
this.getScope() instanceof Module and result = this.getName()
or
exists(string enclosing_name |
enclosing_name = this.getScope().(Function).getQualifiedName()
or
enclosing_name = this.getScope().(Class).getQualifiedName()
|
result = enclosing_name + "." + this.getName()
)
}
override predicate containsInScope(AstNode inner) { Scope.super.containsInScope(inner) }
override predicate contains(AstNode inner) { Scope.super.contains(inner) }
}

104
python/ql/src/semmle/python/Comment.qll
View File

@@ -1,104 +0,0 @@
/**
* Provides classes representing comments in Python.
*/
import python
/** A source code comment */
class Comment extends @py_comment {
/** Gets the full text of the comment including the leading '#' */
string getText() { py_comments(this, result, _) }
/** Gets the contents of the comment excluding the leading '#' */
string getContents() { result = this.getText().suffix(1) }
Location getLocation() { py_comments(this, _, result) }
/** Gets a textual representation of this element. */
string toString() { result = "Comment " + this.getText() }
/**
* Gets this immediately following comment.
* Blanks line are allowed between this comment and the following comment,
* but code or other comments are not.
*/
Comment getFollowing() {
exists(File f, int n | this.file_line(f, n) |
result.file_line(f, n + 1)
or
result.file_line(f, n + 2) and f.emptyLine(n + 1)
or
result.file_line(f, n + 3) and f.emptyLine(n + 2) and f.emptyLine(n + 1)
)
}
private predicate file_line(File f, int n) {
this.getLocation().getFile() = f and
this.getLocation().getStartLine() = n
}
}
private predicate comment_block_part(Comment start, Comment part, int i) {
not exists(Comment prev | prev.getFollowing() = part) and
exists(Comment following | part.getFollowing() = following) and
start = part and
i = 1
or
exists(Comment prev |
comment_block_part(start, prev, i - 1) and
part = prev.getFollowing()
)
}
/** A block of consecutive comments */
class CommentBlock extends @py_comment {
CommentBlock() { comment_block_part(this, _, _) }
private Comment last() { comment_block_part(this, result, this.length()) }
/** Gets a textual representation of this element. */
string toString() { result = "Comment block" }
/** The length of this comment block (in comments) */
int length() { result = max(int i | comment_block_part(this, _, i)) }
/**
* Holds if this element is at the specified location.
* The location spans column `startcolumn` of line `startline` to
* column `endcolumn` of line `endline` in file `filepath`.
* For more information, see
* [Locations](https://help.semmle.com/QL/learn-ql/ql/locations.html).
*/
predicate hasLocationInfo(
string filepath, int startline, int startcolumn, int endline, int endcolumn
) {
this.(Comment).getLocation().hasLocationInfo(filepath, startline, startcolumn, _, _) and
exists(Comment end | end = this.last() |
end.getLocation().hasLocationInfo(_, _, _, endline, endcolumn)
)
}
/** Holds if this comment block contains `c`. */
predicate contains(Comment c) {
comment_block_part(this, c, _)
or
this = c
}
/** Gets a string representation of this comment block. */
string getContents() {
result =
concat(Comment c, int i |
comment_block_part(this, c, i)
or
this = c and i = 0
|
c.getContents() order by i
)
}
}
/** A type-hint comment. Any comment that starts with `# type:` */
class TypeHintComment extends Comment {
TypeHintComment() { this.getText().regexpMatch("# +type:.*") }
}

524
python/ql/src/semmle/python/Comparisons.qll
View File

@@ -1,524 +0,0 @@
/**
* Provides classes representing comparison operators.
*/
import python
/** A class representing the six comparison operators, ==, !=, <, <=, > and >=. */
class CompareOp extends int {
CompareOp() { this in [1 .. 6] }
/** Gets the logical inverse operator */
CompareOp invert() {
this = eq() and result = ne()
or
this = ne() and result = eq()
or
this = lt() and result = ge()
or
this = gt() and result = le()
or
this = le() and result = gt()
or
this = ge() and result = lt()
}
/** Gets the reverse operator (swapping the operands) */
CompareOp reverse() {
this = eq() and result = eq()
or
this = ne() and result = ne()
or
this = lt() and result = gt()
or
this = gt() and result = lt()
or
this = le() and result = ge()
or
this = ge() and result = le()
}
/** Gets the textual representation of `this`. */
string repr() {
this = eq() and result = "=="
or
this = ne() and result = "!="
or
this = lt() and result = "<"
or
this = gt() and result = ">"
or
this = le() and result = "<="
or
this = ge() and result = ">="
}
/** Holds if `op` is the `Cmpop` corresponding to `this`. */
predicate forOp(Cmpop op) {
op instanceof Eq and this = eq()
or
op instanceof NotEq and this = ne()
or
op instanceof Lt and this = lt()
or
op instanceof LtE and this = le()
or
op instanceof Gt and this = gt()
or
op instanceof GtE and this = ge()
}
/** Return this if isTrue is true, otherwise returns the inverse */
CompareOp conditional(boolean isTrue) {
result = this and isTrue = true
or
result = this.invert() and isTrue = false
}
}
/** The `CompareOp` for "equals". */
CompareOp eq() { result = 1 }
/** The `CompareOp` for "not equals". */
CompareOp ne() { result = 2 }
/** The `CompareOp` for "less than". */
CompareOp lt() { result = 3 }
/** The `CompareOp` for "less than or equal to". */
CompareOp le() { result = 4 }
/** The `CompareOp` for "greater than". */
CompareOp gt() { result = 5 }
/** The `CompareOp` for "greater than or equal to". */
CompareOp ge() { result = 6 }
/* Workaround precision limits in floating point numbers */
bindingset[x]
private predicate ok_magnitude(float x) {
x > -9007199254740992.0 and // -2**53
x < 9007199254740992.0 // 2**53
}
bindingset[x, y]
private float add(float x, float y) {
ok_magnitude(x) and
ok_magnitude(y) and
ok_magnitude(result) and
result = x + y
}
bindingset[x, y]
private float sub(float x, float y) {
ok_magnitude(x) and
ok_magnitude(y) and
ok_magnitude(result) and
result = x - y
}
/** Normalise equality cmp into the form `left op right + k`. */
private predicate test(
ControlFlowNode cmp, ControlFlowNode left, CompareOp op, ControlFlowNode right, float k
) {
simple_test(cmp, left, op, right) and k = 0
or
add_test(cmp, left, op, right, k)
or
not_test(cmp, left, op, right, k)
or
subtract_test(cmp, left, op, right, k)
or
exists(float c | test(cmp, right, op.reverse(), left, c) and k = -c)
}
/** Various simple tests in left op right + k form. */
private predicate simple_test(CompareNode cmp, ControlFlowNode l, CompareOp cmpop, ControlFlowNode r) {
exists(Cmpop op | cmp.operands(l, op, r) and cmpop.forOp(op))
}
private predicate add_test_left(
CompareNode cmp, ControlFlowNode l, CompareOp op, ControlFlowNode r, float k
) {
exists(BinaryExprNode lhs, float c, float x, Num n |
lhs.getNode().getOp() instanceof Add and
test(cmp, lhs, op, r, c) and
x = n.getN().toFloat() and
k = sub(c, x)
|
l = lhs.getLeft() and n = lhs.getRight().getNode()
or
l = lhs.getRight() and n = lhs.getLeft().getNode()
)
}
private predicate add_test_right(
CompareNode cmp, ControlFlowNode l, CompareOp op, ControlFlowNode r, float k
) {
exists(BinaryExprNode rhs, float c, float x, Num n |
rhs.getNode().getOp() instanceof Add and
test(cmp, l, op, rhs, c) and
x = n.getN().toFloat() and
k = add(c, x)
|
r = rhs.getLeft() and n = rhs.getRight().getNode()
or
r = rhs.getRight() and n = rhs.getLeft().getNode()
)
}
/*
* left + x op right + c => left op right + (c-x)
* left op (right + x) + c => left op right + (c+x)
*/
private predicate add_test(
CompareNode cmp, ControlFlowNode l, CompareOp op, ControlFlowNode r, float k
) {
add_test_left(cmp, l, op, r, k)
or
add_test_right(cmp, l, op, r, k)
}
private predicate subtract_test_left(
CompareNode cmp, ControlFlowNode l, CompareOp op, ControlFlowNode r, float k
) {
exists(BinaryExprNode lhs, float c, float x, Num n |
lhs.getNode().getOp() instanceof Sub and
test(cmp, lhs, op, r, c) and
l = lhs.getLeft() and
n = lhs.getRight().getNode() and
x = n.getN().toFloat()
|
k = add(c, x)
)
}
private predicate subtract_test_right(
CompareNode cmp, ControlFlowNode l, CompareOp op, ControlFlowNode r, float k
) {
exists(BinaryExprNode rhs, float c, float x, Num n |
rhs.getNode().getOp() instanceof Sub and
test(cmp, l, op, rhs, c) and
r = rhs.getRight() and
n = rhs.getLeft().getNode() and
x = n.getN().toFloat()
|
k = sub(c, x)
)
}
/*
* left - x op right + c => left op right + (c+x)
* left op (right - x) + c => left op right + (c-x)
*/
private predicate subtract_test(
CompareNode cmp, ControlFlowNode l, CompareOp op, ControlFlowNode r, float k
) {
subtract_test_left(cmp, l, op, r, k)
or
subtract_test_right(cmp, l, op, r, k)
}
private predicate not_test(
UnaryExprNode u, ControlFlowNode l, CompareOp op, ControlFlowNode r, float k
) {
u.getNode().getOp() instanceof Not and
test(u.getOperand(), l, op.invert(), r, k)
}
/**
* A comparison which can be simplified to the canonical form `x OP y + k` where `x` and `y` are `ControlFlowNode`s,
* `k` is a floating point constant and `OP` is one of `<=`, `>`, `==` or `!=`.
*/
class Comparison extends ControlFlowNode {
Comparison() { test(this, _, _, _, _) }
/** Whether this condition tests `l op r + k` */
predicate tests(ControlFlowNode l, CompareOp op, ControlFlowNode r, float k) {
test(this, l, op, r, k)
}
/** Whether this condition tests `l op k` */
predicate tests(ControlFlowNode l, CompareOp op, float k) {
exists(ControlFlowNode r, float x, float c | test(this, l, op, r, c) |
x = r.getNode().(Num).getN().toFloat() and
k = add(c, x)
)
}
/*
* The following predicates determine whether this test, when its result is `thisIsTrue`,
* is equivalent to the predicate `v OP k` or `v1 OP v2 + k`.
* For example, the test `x <= y` being false, is equivalent to the predicate `x > y`.
*/
private predicate equivalentToEq(boolean thisIsTrue, SsaVariable v, float k) {
this.tests(v.getAUse(), eq().conditional(thisIsTrue), k)
}
private predicate equivalentToNotEq(boolean thisIsTrue, SsaVariable v, float k) {
this.tests(v.getAUse(), ne().conditional(thisIsTrue), k)
}
private predicate equivalentToLt(boolean thisIsTrue, SsaVariable v, float k) {
this.tests(v.getAUse(), lt().conditional(thisIsTrue), k)
}
private predicate equivalentToLtEq(boolean thisIsTrue, SsaVariable v, float k) {
this.tests(v.getAUse(), le().conditional(thisIsTrue), k)
}
private predicate equivalentToGt(boolean thisIsTrue, SsaVariable v, float k) {
this.tests(v.getAUse(), gt().conditional(thisIsTrue), k)
}
private predicate equivalentToGtEq(boolean thisIsTrue, SsaVariable v, float k) {
this.tests(v.getAUse(), ge().conditional(thisIsTrue), k)
}
private predicate equivalentToEq(boolean thisIsTrue, SsaVariable v1, SsaVariable v2, float k) {
this.tests(v1.getAUse(), eq().conditional(thisIsTrue), v2.getAUse(), k)
}
private predicate equivalentToNotEq(boolean thisIsTrue, SsaVariable v1, SsaVariable v2, float k) {
this.tests(v1.getAUse(), ne().conditional(thisIsTrue), v2.getAUse(), k)
}
private predicate equivalentToLt(boolean thisIsTrue, SsaVariable v1, SsaVariable v2, float k) {
this.tests(v1.getAUse(), lt().conditional(thisIsTrue), v2.getAUse(), k)
}
private predicate equivalentToLtEq(boolean thisIsTrue, SsaVariable v1, SsaVariable v2, float k) {
this.tests(v1.getAUse(), le().conditional(thisIsTrue), v2.getAUse(), k)
}
private predicate equivalentToGt(boolean thisIsTrue, SsaVariable v1, SsaVariable v2, float k) {
this.tests(v1.getAUse(), gt().conditional(thisIsTrue), v2.getAUse(), k)
}
private predicate equivalentToGtEq(boolean thisIsTrue, SsaVariable v1, SsaVariable v2, float k) {
this.tests(v1.getAUse(), ge().conditional(thisIsTrue), v2.getAUse(), k)
}
/**
* Whether the result of this comparison being `thisIsTrue` implies that the result of `that` is `isThatTrue`.
* In other words, does the predicate that is equivalent to the result of `this` being `thisIsTrue`
* imply the predicate that is equivalent to the result of `that` being `thatIsTrue`.
* For example, assume that there are two tests, which when normalised have the form `x < y` and `x > y + 1`.
* Then the test `x < y` having a true result, implies that the test `x > y + 1` will have a false result.
* (`x < y` having a false result implies nothing about `x > y + 1`)
*/
predicate impliesThat(boolean thisIsTrue, Comparison that, boolean thatIsTrue) {
/* `v == k` => `v == k` */
exists(SsaVariable v, float k1, float k2 |
this.equivalentToEq(thisIsTrue, v, k1) and
that.equivalentToEq(thatIsTrue, v, k2) and
eq(k1, k2)
or
this.equivalentToNotEq(thisIsTrue, v, k1) and
that.equivalentToNotEq(thatIsTrue, v, k2) and
eq(k1, k2)
)
or
exists(SsaVariable v, float k1, float k2 |
/* `v < k1` => `v != k2` iff k1 <= k2 */
this.equivalentToLt(thisIsTrue, v, k1) and
that.equivalentToNotEq(thatIsTrue, v, k2) and
le(k1, k2)
or
/* `v <= k1` => `v != k2` iff k1 < k2 */
this.equivalentToLtEq(thisIsTrue, v, k1) and
that.equivalentToNotEq(thatIsTrue, v, k2) and
lt(k1, k2)
or
/* `v > k1` => `v != k2` iff k1 >= k2 */
this.equivalentToGt(thisIsTrue, v, k1) and
that.equivalentToNotEq(thatIsTrue, v, k2) and
ge(k1, k2)
or
/* `v >= k1` => `v != k2` iff k1 > k2 */
this.equivalentToGtEq(thisIsTrue, v, k1) and
that.equivalentToNotEq(thatIsTrue, v, k2) and
gt(k1, k2)
)
or
exists(SsaVariable v, float k1, float k2 |
/* `v < k1` => `v < k2` iff k1 <= k2 */
this.equivalentToLt(thisIsTrue, v, k1) and
that.equivalentToLt(thatIsTrue, v, k2) and
le(k1, k2)
or
/* `v < k1` => `v <= k2` iff k1 <= k2 */
this.equivalentToLt(thisIsTrue, v, k1) and
that.equivalentToLtEq(thatIsTrue, v, k2) and
le(k1, k2)
or
/* `v <= k1` => `v < k2` iff k1 < k2 */
this.equivalentToLtEq(thisIsTrue, v, k1) and
that.equivalentToLt(thatIsTrue, v, k2) and
lt(k1, k2)
or
/* `v <= k1` => `v <= k2` iff k1 <= k2 */
this.equivalentToLtEq(thisIsTrue, v, k1) and
that.equivalentToLtEq(thatIsTrue, v, k2) and
le(k1, k2)
)
or
exists(SsaVariable v, float k1, float k2 |
/* `v > k1` => `v >= k2` iff k1 >= k2 */
this.equivalentToGt(thisIsTrue, v, k1) and
that.equivalentToGt(thatIsTrue, v, k2) and
ge(k1, k2)
or
/* `v > k1` => `v >= k2` iff k1 >= k2 */
this.equivalentToGt(thisIsTrue, v, k1) and
that.equivalentToGtEq(thatIsTrue, v, k2) and
ge(k1, k2)
or
/* `v >= k1` => `v > k2` iff k1 > k2 */
this.equivalentToGtEq(thisIsTrue, v, k1) and
that.equivalentToGt(thatIsTrue, v, k2) and
gt(k1, k2)
or
/* `v >= k1` => `v >= k2` iff k1 >= k2 */
this.equivalentToGtEq(thisIsTrue, v, k1) and
that.equivalentToGtEq(thatIsTrue, v, k2) and
ge(k1, k2)
)
or
exists(SsaVariable v1, SsaVariable v2, float k |
/* `v1 == v2 + k` => `v1 == v2 + k` */
this.equivalentToEq(thisIsTrue, v1, v2, k) and
that.equivalentToEq(thatIsTrue, v1, v2, k)
or
this.equivalentToNotEq(thisIsTrue, v1, v2, k) and
that.equivalentToNotEq(thatIsTrue, v1, v2, k)
)
or
exists(SsaVariable v1, SsaVariable v2, float k1, float k2 |
/* `v1 < v2 + k1` => `v1 != v2 + k2` iff k1 <= k2 */
this.equivalentToLt(thisIsTrue, v1, v2, k1) and
that.equivalentToNotEq(thatIsTrue, v1, v2, k2) and
le(k1, k2)
or
/* `v1 <= v2 + k1` => `v1 != v2 + k2` iff k1 < k2 */
this.equivalentToLtEq(thisIsTrue, v1, v2, k1) and
that.equivalentToNotEq(thatIsTrue, v1, v2, k2) and
lt(k1, k2)
or
/* `v1 > v2 + k1` => `v1 != v2 + k2` iff k1 >= k2 */
this.equivalentToGt(thisIsTrue, v1, v2, k1) and
that.equivalentToNotEq(thatIsTrue, v1, v2, k2) and
ge(k1, k2)
or
/* `v1 >= v2 + k1` => `v1 != v2 + k2` iff k1 > k2 */
this.equivalentToGtEq(thisIsTrue, v1, v2, k1) and
that.equivalentToNotEq(thatIsTrue, v1, v2, k2) and
gt(k1, k2)
)
or
exists(SsaVariable v1, SsaVariable v2, float k1, float k2 |
/* `v1 <= v2 + k1` => `v1 <= v2 + k2` iff k1 <= k2 */
this.equivalentToLtEq(thisIsTrue, v1, v2, k1) and
that.equivalentToLtEq(thatIsTrue, v1, v2, k2) and
le(k1, k2)
or
/* `v1 < v2 + k1` => `v1 <= v2 + k2` iff k1 <= k2 */
this.equivalentToLt(thisIsTrue, v1, v2, k1) and
that.equivalentToLtEq(thatIsTrue, v1, v2, k2) and
le(k1, k2)
or
/* `v1 <= v2 + k1` => `v1 < v2 + k2` iff k1 < k2 */
this.equivalentToLtEq(thisIsTrue, v1, v2, k1) and
that.equivalentToLt(thatIsTrue, v1, v2, k2) and
lt(k1, k2)
or
/* `v1 <= v2 + k1` => `v1 <= v2 + k2` iff k1 <= k2 */
this.equivalentToLtEq(thisIsTrue, v1, v2, k1) and
that.equivalentToLtEq(thatIsTrue, v1, v2, k2) and
le(k1, k2)
)
or
exists(SsaVariable v1, SsaVariable v2, float k1, float k2 |
/* `v1 > v2 + k1` => `v1 > v2 + k2` iff k1 >= k2 */
this.equivalentToGt(thisIsTrue, v1, v2, k1) and
that.equivalentToGt(thatIsTrue, v1, v2, k2) and
ge(k1, k2)
or
/* `v1 > v2 + k1` => `v2 >= v2 + k2` iff k1 >= k2 */
this.equivalentToGt(thisIsTrue, v1, v2, k1) and
that.equivalentToGtEq(thatIsTrue, v1, v2, k2) and
ge(k1, k2)
or
/* `v1 >= v2 + k1` => `v2 > v2 + k2` iff k1 > k2 */
this.equivalentToGtEq(thisIsTrue, v1, v2, k1) and
that.equivalentToGt(thatIsTrue, v1, v2, k2) and
gt(k1, k2)
or
/* `v1 >= v2 + k1` => `v2 >= v2 + k2` iff k1 >= k2 */
this.equivalentToGtEq(thisIsTrue, v1, v2, k1) and
that.equivalentToGtEq(thatIsTrue, v1, v2, k2) and
ge(k1, k2)
)
}
}
/* Work around differences in floating-point comparisons between Python and QL */
private predicate is_zero(float x) {
x = 0.0
or
x = -0.0
}
bindingset[x, y]
private predicate lt(float x, float y) { if is_zero(x) then y > 0 else x < y }
bindingset[x, y]
private predicate eq(float x, float y) { if is_zero(x) then is_zero(y) else x = y }
bindingset[x, y]
private predicate gt(float x, float y) { lt(y, x) }
bindingset[x, y]
private predicate le(float x, float y) { lt(x, y) or eq(x, y) }
bindingset[x, y]
private predicate ge(float x, float y) { lt(y, x) or eq(x, y) }
/**
* A basic block which terminates in a condition, splitting the subsequent control flow,
* in which the condition is an instance of `Comparison`
*/
class ComparisonControlBlock extends ConditionBlock {
ComparisonControlBlock() { this.getLastNode() instanceof Comparison }
/** Whether this conditional guard determines that, in block `b`, `l == r + k` if `eq` is true, or `l != r + k` if `eq` is false, */
predicate controls(ControlFlowNode l, CompareOp op, ControlFlowNode r, float k, BasicBlock b) {
exists(boolean control |
this.controls(b, control) and this.getTest().tests(l, op, r, k) and control = true
or
this.controls(b, control) and this.getTest().tests(l, op.invert(), r, k) and control = false
)
}
/** Whether this conditional guard determines that, in block `b`, `l == r + k` if `eq` is true, or `l != r + k` if `eq` is false, */
predicate controls(ControlFlowNode l, CompareOp op, float k, BasicBlock b) {
exists(boolean control |
this.controls(b, control) and this.getTest().tests(l, op, k) and control = true
or
this.controls(b, control) and this.getTest().tests(l, op.invert(), k) and control = false
)
}
Comparison getTest() { this.getLastNode() = result }
/** Whether this conditional guard implies that, in block `b`, the result of `that` is `thatIsTrue` */
predicate impliesThat(BasicBlock b, Comparison that, boolean thatIsTrue) {
exists(boolean controlSense |
this.controls(b, controlSense) and
this.getTest().impliesThat(controlSense, that, thatIsTrue)
)
}
}

122
python/ql/src/semmle/python/Comprehensions.qll
View File

@@ -1,122 +0,0 @@
import python
/** Base class for list, set and dictionary comprehensions, and generator expressions. */
abstract class Comp extends Expr {
abstract Function getFunction();
/** Gets the iterable of this set comprehension. */
abstract Expr getIterable();
/** Gets the iteration variable for the nth innermost generator of this comprehension. */
Variable getIterationVariable(int n) {
result.getAnAccess() = this.getNthInnerLoop(n).getTarget()
}
/** Gets the nth innermost For expression of this comprehension. */
For getNthInnerLoop(int n) {
n = 0 and result = this.getFunction().getStmt(0)
or
result = this.getNthInnerLoop(n - 1).getStmt(0)
}
/** Gets the iteration variable for a generator of this list comprehension. */
Variable getAnIterationVariable() { result = this.getIterationVariable(_) }
/** Gets the scope in which the body of this list comprehension evaluates. */
Scope getEvaluatingScope() { result = this.getFunction() }
/** Gets the expression for elements of this comprehension. */
Expr getElt() {
exists(Yield yield, Stmt body |
result = yield.getValue() and
body = this.getNthInnerLoop(_).getAStmt()
|
yield = body.(ExprStmt).getValue()
or
yield = body.(If).getStmt(0).(ExprStmt).getValue()
)
}
}
/** A list comprehension, such as `[ chr(x) for x in range(ord('A'), ord('Z')+1) ]` */
class ListComp extends ListComp_, Comp {
override Expr getASubExpression() {
result = this.getAGenerator().getASubExpression() or
result = this.getElt() or
result = this.getIterable()
}
override AstNode getAChildNode() {
result = this.getAGenerator() or
result = this.getIterable() or
result = this.getFunction()
}
override predicate hasSideEffects() { any() }
/** Gets the scope in which the body of this list comprehension evaluates. */
override Scope getEvaluatingScope() {
major_version() = 2 and result = this.getScope()
or
major_version() = 3 and result = this.getFunction()
}
/** Gets the iteration variable for the nth innermost generator of this list comprehension */
override Variable getIterationVariable(int n) { result = Comp.super.getIterationVariable(n) }
override Function getFunction() { result = ListComp_.super.getFunction() }
override Expr getIterable() { result = ListComp_.super.getIterable() }
override string toString() { result = ListComp_.super.toString() }
override Expr getElt() { result = Comp.super.getElt() }
}
/** A set comprehension such as `{ v for v in "0123456789" }` */
class SetComp extends SetComp_, Comp {
override Expr getASubExpression() { result = this.getIterable() }
override AstNode getAChildNode() {
result = this.getASubExpression() or
result = this.getFunction()
}
override predicate hasSideEffects() { any() }
override Function getFunction() { result = SetComp_.super.getFunction() }
override Expr getIterable() { result = SetComp_.super.getIterable() }
}
/** A dictionary comprehension, such as `{ k:v for k, v in enumerate("0123456789") }` */
class DictComp extends DictComp_, Comp {
override Expr getASubExpression() { result = this.getIterable() }
override AstNode getAChildNode() {
result = this.getASubExpression() or
result = this.getFunction()
}
override predicate hasSideEffects() { any() }
override Function getFunction() { result = DictComp_.super.getFunction() }
override Expr getIterable() { result = DictComp_.super.getIterable() }
}
/** A generator expression, such as `(var for var in iterable)` */
class GeneratorExp extends GeneratorExp_, Comp {
override Expr getASubExpression() { result = this.getIterable() }
override AstNode getAChildNode() {
result = this.getASubExpression() or
result = this.getFunction()
}
override predicate hasSideEffects() { any() }
override Function getFunction() { result = GeneratorExp_.super.getFunction() }
override Expr getIterable() { result = GeneratorExp_.super.getIterable() }
}

863
python/ql/src/semmle/python/Concepts.qll
View File

@@ -1,863 +0,0 @@
/**
* Provides abstract classes representing generic concepts such as file system
* access or system command execution, for which individual framework libraries
* provide concrete subclasses.
*/
private import python
private import semmle.python.dataflow.new.DataFlow
private import semmle.python.dataflow.new.RemoteFlowSources
private import semmle.python.dataflow.new.TaintTracking
private import semmle.python.Frameworks
/**
* A data-flow node that executes an operating system command,
* for instance by spawning a new process.
*
* Extend this class to refine existing API models. If you want to model new APIs,
* extend `SystemCommandExecution::Range` instead.
*/
class SystemCommandExecution extends DataFlow::Node {
SystemCommandExecution::Range range;
SystemCommandExecution() { this = range }
/** Gets the argument that specifies the command to be executed. */
DataFlow::Node getCommand() { result = range.getCommand() }
}
/** Provides a class for modeling new system-command execution APIs. */
module SystemCommandExecution {
/**
* A data-flow node that executes an operating system command,
* for instance by spawning a new process.
*
* Extend this class to model new APIs. If you want to refine existing API models,
* extend `SystemCommandExecution` instead.
*/
abstract class Range extends DataFlow::Node {
/** Gets the argument that specifies the command to be executed. */
abstract DataFlow::Node getCommand();
}
}
/**
* A data flow node that performs a file system access, including reading and writing data,
* creating and deleting files and folders, checking and updating permissions, and so on.
*
* Extend this class to refine existing API models. If you want to model new APIs,
* extend `FileSystemAccess::Range` instead.
*/
class FileSystemAccess extends DataFlow::Node {
FileSystemAccess::Range range;
FileSystemAccess() { this = range }
/** Gets an argument to this file system access that is interpreted as a path. */
DataFlow::Node getAPathArgument() { result = range.getAPathArgument() }
}
/** Provides a class for modeling new file system access APIs. */
module FileSystemAccess {
/**
* A data-flow node that performs a file system access, including reading and writing data,
* creating and deleting files and folders, checking and updating permissions, and so on.
*
* Extend this class to model new APIs. If you want to refine existing API models,
* extend `FileSystemAccess` instead.
*/
abstract class Range extends DataFlow::Node {
/** Gets an argument to this file system access that is interpreted as a path. */
abstract DataFlow::Node getAPathArgument();
}
}
/**
* A data flow node that writes data to the file system access.
*
* Extend this class to refine existing API models. If you want to model new APIs,
* extend `FileSystemWriteAccess::Range` instead.
*/
class FileSystemWriteAccess extends FileSystemAccess {
override FileSystemWriteAccess::Range range;
/**
* Gets a node that represents data to be written to the file system (possibly with
* some transformation happening before it is written, like JSON encoding).
*/
DataFlow::Node getADataNode() { result = range.getADataNode() }
}
/** Provides a class for modeling new file system writes. */
module FileSystemWriteAccess {
/**
* A data flow node that writes data to the file system access.
*
* Extend this class to model new APIs. If you want to refine existing API models,
* extend `FileSystemWriteAccess` instead.
*/
abstract class Range extends FileSystemAccess::Range {
/**
* Gets a node that represents data to be written to the file system (possibly with
* some transformation happening before it is written, like JSON encoding).
*/
abstract DataFlow::Node getADataNode();
}
}
/** Provides classes for modeling path-related APIs. */
module Path {
/**
* A data-flow node that performs path normalization. This is often needed in order
* to safely access paths.
*/
class PathNormalization extends DataFlow::Node {
PathNormalization::Range range;
PathNormalization() { this = range }
}
/** Provides a class for modeling new path normalization APIs. */
module PathNormalization {
/**
* A data-flow node that performs path normalization. This is often needed in order
* to safely access paths.
*/
abstract class Range extends DataFlow::Node { }
}
/** A data-flow node that checks that a path is safe to access. */
class SafeAccessCheck extends DataFlow::BarrierGuard {
SafeAccessCheck::Range range;
SafeAccessCheck() { this = range }
override predicate checks(ControlFlowNode node, boolean branch) { range.checks(node, branch) }
}
/** Provides a class for modeling new path safety checks. */
module SafeAccessCheck {
/** A data-flow node that checks that a path is safe to access. */
abstract class Range extends DataFlow::BarrierGuard { }
}
}
/**
* A data-flow node that decodes data from a binary or textual format. This
* is intended to include deserialization, unmarshalling, decoding, unpickling,
* decompressing, decrypting, parsing etc.
*
* A decoding (automatically) preserves taint from input to output. However, it can
* also be a problem in itself, for example if it allows code execution or could result
* in denial-of-service.
*
* Extend this class to refine existing API models. If you want to model new APIs,
* extend `Decoding::Range` instead.
*/
class Decoding extends DataFlow::Node {
Decoding::Range range;
Decoding() { this = range }
/** Holds if this call may execute code embedded in its input. */
predicate mayExecuteInput() { range.mayExecuteInput() }
/** Gets an input that is decoded by this function. */
DataFlow::Node getAnInput() { result = range.getAnInput() }
/** Gets the output that contains the decoded data produced by this function. */
DataFlow::Node getOutput() { result = range.getOutput() }
/** Gets an identifier for the format this function decodes from, such as "JSON". */
string getFormat() { result = range.getFormat() }
}
/** Provides a class for modeling new decoding mechanisms. */
module Decoding {
/**
* A data-flow node that decodes data from a binary or textual format. This
* is intended to include deserialization, unmarshalling, decoding, unpickling,
* decompressing, decrypting, parsing etc.
*
* A decoding (automatically) preserves taint from input to output. However, it can
* also be a problem in itself, for example if it allows code execution or could result
* in denial-of-service.
*
* Extend this class to model new APIs. If you want to refine existing API models,
* extend `Decoding` instead.
*/
abstract class Range extends DataFlow::Node {
/** Holds if this call may execute code embedded in its input. */
abstract predicate mayExecuteInput();
/** Gets an input that is decoded by this function. */
abstract DataFlow::Node getAnInput();
/** Gets the output that contains the decoded data produced by this function. */
abstract DataFlow::Node getOutput();
/** Gets an identifier for the format this function decodes from, such as "JSON". */
abstract string getFormat();
}
}
private class DecodingAdditionalTaintStep extends TaintTracking::AdditionalTaintStep {
override predicate step(DataFlow::Node nodeFrom, DataFlow::Node nodeTo) {
exists(Decoding decoding |
nodeFrom = decoding.getAnInput() and
nodeTo = decoding.getOutput()
)
}
}
/**
* A data-flow node that encodes data to a binary or textual format. This
* is intended to include serialization, marshalling, encoding, pickling,
* compressing, encrypting, etc.
*
* An encoding (automatically) preserves taint from input to output.
*
* Extend this class to refine existing API models. If you want to model new APIs,
* extend `Encoding::Range` instead.
*/
class Encoding extends DataFlow::Node {
Encoding::Range range;
Encoding() { this = range }
/** Gets an input that is encoded by this function. */
DataFlow::Node getAnInput() { result = range.getAnInput() }
/** Gets the output that contains the encoded data produced by this function. */
DataFlow::Node getOutput() { result = range.getOutput() }
/** Gets an identifier for the format this function decodes from, such as "JSON". */
string getFormat() { result = range.getFormat() }
}
/** Provides a class for modeling new encoding mechanisms. */
module Encoding {
/**
* A data-flow node that encodes data to a binary or textual format. This
* is intended to include serialization, marshalling, encoding, pickling,
* compressing, encrypting, etc.
*
* An encoding (automatically) preserves taint from input to output.
*
* Extend this class to model new APIs. If you want to refine existing API models,
* extend `Encoding` instead.
*/
abstract class Range extends DataFlow::Node {
/** Gets an input that is encoded by this function. */
abstract DataFlow::Node getAnInput();
/** Gets the output that contains the encoded data produced by this function. */
abstract DataFlow::Node getOutput();
/** Gets an identifier for the format this function decodes from, such as "JSON". */
abstract string getFormat();
}
}
private class EncodingAdditionalTaintStep extends TaintTracking::AdditionalTaintStep {
override predicate step(DataFlow::Node nodeFrom, DataFlow::Node nodeTo) {
exists(Encoding encoding |
nodeFrom = encoding.getAnInput() and
nodeTo = encoding.getOutput()
)
}
}
/**
* A data-flow node that logs data.
*
* Extend this class to refine existing API models. If you want to model new APIs,
* extend `Logging::Range` instead.
*/
class Logging extends DataFlow::Node {
Logging::Range range;
Logging() { this = range }
/** Gets an input that is logged. */
DataFlow::Node getAnInput() { result = range.getAnInput() }
}
/** Provides a class for modeling new logging mechanisms. */
module Logging {
/**
* A data-flow node that logs data.
*
* Extend this class to model new APIs. If you want to refine existing API models,
* extend `Logging` instead.
*/
abstract class Range extends DataFlow::Node {
/** Gets an input that is logged. */
abstract DataFlow::Node getAnInput();
}
}
/**
* A data-flow node that dynamically executes Python code.
*
* Extend this class to refine existing API models. If you want to model new APIs,
* extend `CodeExecution::Range` instead.
*/
class CodeExecution extends DataFlow::Node {
CodeExecution::Range range;
CodeExecution() { this = range }
/** Gets the argument that specifies the code to be executed. */
DataFlow::Node getCode() { result = range.getCode() }
}
/** Provides a class for modeling new dynamic code execution APIs. */
module CodeExecution {
/**
* A data-flow node that dynamically executes Python code.
*
* Extend this class to model new APIs. If you want to refine existing API models,
* extend `CodeExecution` instead.
*/
abstract class Range extends DataFlow::Node {
/** Gets the argument that specifies the code to be executed. */
abstract DataFlow::Node getCode();
}
}
/**
* A data-flow node that executes SQL statements.
*
* Extend this class to refine existing API models. If you want to model new APIs,
* extend `SqlExecution::Range` instead.
*/
class SqlExecution extends DataFlow::Node {
SqlExecution::Range range;
SqlExecution() { this = range }
/** Gets the argument that specifies the SQL statements to be executed. */
DataFlow::Node getSql() { result = range.getSql() }
}
/** Provides a class for modeling new SQL execution APIs. */
module SqlExecution {
/**
* A data-flow node that executes SQL statements.
*
* Extend this class to model new APIs. If you want to refine existing API models,
* extend `SqlExecution` instead.
*/
abstract class Range extends DataFlow::Node {
/** Gets the argument that specifies the SQL statements to be executed. */
abstract DataFlow::Node getSql();
}
}
/**
* A data-flow node that escapes meta-characters, which could be used to prevent
* injection attacks.
*
* Extend this class to refine existing API models. If you want to model new APIs,
* extend `Escaping::Range` instead.
*/
class Escaping extends DataFlow::Node {
Escaping::Range range;
Escaping() {
this = range and
// escapes that don't have _both_ input/output defined are not valid
exists(range.getAnInput()) and
exists(range.getOutput())
}
/** Gets an input that will be escaped. */
DataFlow::Node getAnInput() { result = range.getAnInput() }
/** Gets the output that contains the escaped data. */
DataFlow::Node getOutput() { result = range.getOutput() }
/**
* Gets the context that this function escapes for, such as `html`, or `url`.
*/
string getKind() { result = range.getKind() }
}
/** Provides a class for modeling new escaping APIs. */
module Escaping {
/**
* A data-flow node that escapes meta-characters, which could be used to prevent
* injection attacks.
*
* Extend this class to model new APIs. If you want to refine existing API models,
* extend `Escaping` instead.
*/
abstract class Range extends DataFlow::Node {
/** Gets an input that will be escaped. */
abstract DataFlow::Node getAnInput();
/** Gets the output that contains the escaped data. */
abstract DataFlow::Node getOutput();
/**
* Gets the context that this function escapes for.
*
* While kinds are represented as strings, this should not be relied upon. Use the
* predicates in the `Escaping` module, such as `getHtmlKind`.
*/
abstract string getKind();
}
/** Gets the escape-kind for escaping a string so it can safely be included in HTML. */
string getHtmlKind() { result = "html" }
// TODO: If adding an XML kind, update the modeling of the `MarkupSafe` PyPI package.
//
// Technically it claims to escape for both HTML and XML, but for now we don't have
// anything that relies on XML escaping, so I'm going to defer deciding whether they
// should be the same kind, or whether they deserve to be treated differently.
}
/**
* An escape of a string so it can be safely included in
* the body of an HTML element, for example, replacing `{}` in
* `<p>{}</p>`.
*/
class HtmlEscaping extends Escaping {
HtmlEscaping() { range.getKind() = Escaping::getHtmlKind() }
}
/** Provides classes for modeling HTTP-related APIs. */
module HTTP {
import semmle.python.web.HttpConstants
/** Provides classes for modeling HTTP servers. */
module Server {
/**
* A data-flow node that sets up a route on a server.
*
* Extend this class to refine existing API models. If you want to model new APIs,
* extend `RouteSetup::Range` instead.
*/
class RouteSetup extends DataFlow::Node {
RouteSetup::Range range;
RouteSetup() { this = range }
/** Gets the URL pattern for this route, if it can be statically determined. */
string getUrlPattern() { result = range.getUrlPattern() }
/**
* Gets a function that will handle incoming requests for this route, if any.
*
* NOTE: This will be modified in the near future to have a `RequestHandler` result, instead of a `Function`.
*/
Function getARequestHandler() { result = range.getARequestHandler() }
/**
* Gets a parameter that will receive parts of the url when handling incoming
* requests for this route, if any. These automatically become a `RemoteFlowSource`.
*/
Parameter getARoutedParameter() { result = range.getARoutedParameter() }
/** Gets a string that identifies the framework used for this route setup. */
string getFramework() { result = range.getFramework() }
}
/** Provides a class for modeling new HTTP routing APIs. */
module RouteSetup {
/**
* A data-flow node that sets up a route on a server.
*
* Extend this class to model new APIs. If you want to refine existing API models,
* extend `RouteSetup` instead.
*/
abstract class Range extends DataFlow::Node {
/** Gets the argument used to set the URL pattern. */
abstract DataFlow::Node getUrlPatternArg();
/** Gets the URL pattern for this route, if it can be statically determined. */
string getUrlPattern() {
exists(StrConst str |
this.getUrlPatternArg().getALocalSource() = DataFlow::exprNode(str) and
result = str.getText()
)
}
/**
* Gets a function that will handle incoming requests for this route, if any.
*
* NOTE: This will be modified in the near future to have a `RequestHandler` result, instead of a `Function`.
*/
abstract Function getARequestHandler();
/**
* Gets a parameter that will receive parts of the url when handling incoming
* requests for this route, if any. These automatically become a `RemoteFlowSource`.
*/
abstract Parameter getARoutedParameter();
/** Gets a string that identifies the framework used for this route setup. */
abstract string getFramework();
}
}
/**
* A function that will handle incoming HTTP requests.
*
* Extend this class to refine existing API models. If you want to model new APIs,
* extend `RequestHandler::Range` instead.
*/
class RequestHandler extends Function {
RequestHandler::Range range;
RequestHandler() { this = range }
/**
* Gets a parameter that could receive parts of the url when handling incoming
* requests, if any. These automatically become a `RemoteFlowSource`.
*/
Parameter getARoutedParameter() { result = range.getARoutedParameter() }
/** Gets a string that identifies the framework used for this route setup. */
string getFramework() { result = range.getFramework() }
}
/** Provides a class for modeling new HTTP request handlers. */
module RequestHandler {
/**
* A function that will handle incoming HTTP requests.
*
* Extend this class to model new APIs. If you want to refine existing API models,
* extend `RequestHandler` instead.
*
* Only extend this class if you can't provide a `RouteSetup`, since we handle that case automatically.
*/
abstract class Range extends Function {
/**
* Gets a parameter that could receive parts of the url when handling incoming
* requests, if any. These automatically become a `RemoteFlowSource`.
*/
abstract Parameter getARoutedParameter();
/** Gets a string that identifies the framework used for this request handler. */
abstract string getFramework();
}
}
private class RequestHandlerFromRouteSetup extends RequestHandler::Range {
RouteSetup rs;
RequestHandlerFromRouteSetup() { this = rs.getARequestHandler() }
override Parameter getARoutedParameter() {
result = rs.getARoutedParameter() and
result in [this.getArg(_), this.getArgByName(_)]
}
override string getFramework() { result = rs.getFramework() }
}
/** A parameter that will receive parts of the url when handling an incoming request. */
private class RoutedParameter extends RemoteFlowSource::Range, DataFlow::ParameterNode {
RequestHandler handler;
RoutedParameter() { this.getParameter() = handler.getARoutedParameter() }
override string getSourceType() { result = handler.getFramework() + " RoutedParameter" }
}
/**
* A data-flow node that creates a HTTP response on a server.
*
* Note: we don't require that this response must be sent to a client (a kind of
* "if a tree falls in a forest and nobody hears it" situation).
*
* Extend this class to refine existing API models. If you want to model new APIs,
* extend `HttpResponse::Range` instead.
*/
class HttpResponse extends DataFlow::Node {
HttpResponse::Range range;
HttpResponse() { this = range }
/** Gets the data-flow node that specifies the body of this HTTP response. */
DataFlow::Node getBody() { result = range.getBody() }
/** Gets the mimetype of this HTTP response, if it can be statically determined. */
string getMimetype() { result = range.getMimetype() }
}
/** Provides a class for modeling new HTTP response APIs. */
module HttpResponse {
/**
* A data-flow node that creates a HTTP response on a server.
*
* Note: we don't require that this response must be sent to a client (a kind of
* "if a tree falls in a forest and nobody hears it" situation).
*
* Extend this class to model new APIs. If you want to refine existing API models,
* extend `HttpResponse` instead.
*/
abstract class Range extends DataFlow::Node {
/** Gets the data-flow node that specifies the body of this HTTP response. */
abstract DataFlow::Node getBody();
/** Gets the data-flow node that specifies the content-type/mimetype of this HTTP response, if any. */
abstract DataFlow::Node getMimetypeOrContentTypeArg();
/** Gets the default mimetype that should be used if `getMimetypeOrContentTypeArg` has no results. */
abstract string getMimetypeDefault();
/** Gets the mimetype of this HTTP response, if it can be statically determined. */
string getMimetype() {
exists(StrConst str |
this.getMimetypeOrContentTypeArg().getALocalSource() = DataFlow::exprNode(str) and
result = str.getText().splitAt(";", 0)
)
or
not exists(this.getMimetypeOrContentTypeArg()) and
result = this.getMimetypeDefault()
}
}
}
/**
* A data-flow node that creates a HTTP redirect response on a server.
*
* Note: we don't require that this redirect must be sent to a client (a kind of
* "if a tree falls in a forest and nobody hears it" situation).
*
* Extend this class to refine existing API models. If you want to model new APIs,
* extend `HttpRedirectResponse::Range` instead.
*/
class HttpRedirectResponse extends HttpResponse {
override HttpRedirectResponse::Range range;
HttpRedirectResponse() { this = range }
/** Gets the data-flow node that specifies the location of this HTTP redirect response. */
DataFlow::Node getRedirectLocation() { result = range.getRedirectLocation() }
}
/** Provides a class for modeling new HTTP redirect response APIs. */
module HttpRedirectResponse {
/**
* A data-flow node that creates a HTTP redirect response on a server.
*
* Note: we don't require that this redirect must be sent to a client (a kind of
* "if a tree falls in a forest and nobody hears it" situation).
*
* Extend this class to model new APIs. If you want to refine existing API models,
* extend `HttpResponse` instead.
*/
abstract class Range extends HTTP::Server::HttpResponse::Range {
/** Gets the data-flow node that specifies the location of this HTTP redirect response. */
abstract DataFlow::Node getRedirectLocation();
}
}
/**
* A data-flow node that sets a cookie in an HTTP response.
*
* Extend this class to refine existing API models. If you want to model new APIs,
* extend `HTTP::CookieWrite::Range` instead.
*/
class CookieWrite extends DataFlow::Node {
CookieWrite::Range range;
CookieWrite() { this = range }
/**
* Gets the argument, if any, specifying the raw cookie header.
*/
DataFlow::Node getHeaderArg() { result = range.getHeaderArg() }
/**
* Gets the argument, if any, specifying the cookie name.
*/
DataFlow::Node getNameArg() { result = range.getNameArg() }
/**
* Gets the argument, if any, specifying the cookie value.
*/
DataFlow::Node getValueArg() { result = range.getValueArg() }
}
/** Provides a class for modeling new cookie writes on HTTP responses. */
module CookieWrite {
/**
* A data-flow node that sets a cookie in an HTTP response.
*
* Note: we don't require that this redirect must be sent to a client (a kind of
* "if a tree falls in a forest and nobody hears it" situation).
*
* Extend this class to model new APIs. If you want to refine existing API models,
* extend `HttpResponse` instead.
*/
abstract class Range extends DataFlow::Node {
/**
* Gets the argument, if any, specifying the raw cookie header.
*/
abstract DataFlow::Node getHeaderArg();
/**
* Gets the argument, if any, specifying the cookie name.
*/
abstract DataFlow::Node getNameArg();
/**
* Gets the argument, if any, specifying the cookie value.
*/
abstract DataFlow::Node getValueArg();
}
}
}
}
/**
* Provides models for cryptographic things.
*
* Note: The `CryptographicAlgorithm` class currently doesn't take weak keys into
* consideration for the `isWeak` member predicate. So RSA is always considered
* secure, although using a low number of bits will actually make it insecure. We plan
* to improve our libraries in the future to more precisely capture this aspect.
*/
module Cryptography {
/** Provides models for public-key cryptography, also called asymmetric cryptography. */
module PublicKey {
/**
* A data-flow node that generates a new key-pair for use with public-key cryptography.
*
* Extend this class to refine existing API models. If you want to model new APIs,
* extend `KeyGeneration::Range` instead.
*/
class KeyGeneration extends DataFlow::Node {
KeyGeneration::Range range;
KeyGeneration() { this = range }
/** Gets the name of the cryptographic algorithm (for example `"RSA"` or `"AES"`). */
string getName() { result = range.getName() }
/** Gets the argument that specifies the size of the key in bits, if available. */
DataFlow::Node getKeySizeArg() { result = range.getKeySizeArg() }
/**
* Gets the size of the key generated (in bits), as well as the `origin` that
* explains how we obtained this specific key size.
*/
int getKeySizeWithOrigin(DataFlow::Node origin) {
result = range.getKeySizeWithOrigin(origin)
}
/** Gets the minimum key size (in bits) for this algorithm to be considered secure. */
int minimumSecureKeySize() { result = range.minimumSecureKeySize() }
}
/** Provides classes for modeling new key-pair generation APIs. */
module KeyGeneration {
/** Gets a back-reference to the keysize argument `arg` that was used to generate a new key-pair. */
private DataFlow::TypeTrackingNode keysizeBacktracker(
DataFlow::TypeBackTracker t, DataFlow::Node arg
) {
t.start() and
arg = any(KeyGeneration::Range r).getKeySizeArg() and
result = arg.getALocalSource()
or
exists(DataFlow::TypeBackTracker t2 | result = keysizeBacktracker(t2, arg).backtrack(t2, t))
}
/** Gets a back-reference to the keysize argument `arg` that was used to generate a new key-pair. */
DataFlow::LocalSourceNode keysizeBacktracker(DataFlow::Node arg) {
result = keysizeBacktracker(DataFlow::TypeBackTracker::end(), arg)
}
/**
* A data-flow node that generates a new key-pair for use with public-key cryptography.
*
* Extend this class to model new APIs. If you want to refine existing API models,
* extend `KeyGeneration` instead.
*/
abstract class Range extends DataFlow::Node {
/** Gets the name of the cryptographic algorithm (for example `"RSA"`). */
abstract string getName();
/** Gets the argument that specifies the size of the key in bits, if available. */
abstract DataFlow::Node getKeySizeArg();
/**
* Gets the size of the key generated (in bits), as well as the `origin` that
* explains how we obtained this specific key size.
*/
int getKeySizeWithOrigin(DataFlow::Node origin) {
origin = keysizeBacktracker(this.getKeySizeArg()) and
result = origin.asExpr().(IntegerLiteral).getValue()
}
/** Gets the minimum key size (in bits) for this algorithm to be considered secure. */
abstract int minimumSecureKeySize();
}
/** A data-flow node that generates a new RSA key-pair. */
abstract class RsaRange extends Range {
final override string getName() { result = "RSA" }
final override int minimumSecureKeySize() { result = 2048 }
}
/** A data-flow node that generates a new DSA key-pair. */
abstract class DsaRange extends Range {
final override string getName() { result = "DSA" }
final override int minimumSecureKeySize() { result = 2048 }
}
/** A data-flow node that generates a new ECC key-pair. */
abstract class EccRange extends Range {
final override string getName() { result = "ECC" }
final override int minimumSecureKeySize() { result = 224 }
}
}
}
import semmle.python.concepts.CryptoAlgorithms
/**
* A data-flow node that is an application of a cryptographic algorithm. For example,
* encryption, decryption, signature-validation.
*
* Extend this class to refine existing API models. If you want to model new APIs,
* extend `CryptographicOperation::Range` instead.
*/
class CryptographicOperation extends DataFlow::Node {
CryptographicOperation::Range range;
CryptographicOperation() { this = range }
/** Gets the algorithm used, if it matches a known `CryptographicAlgorithm`. */
CryptographicAlgorithm getAlgorithm() { result = range.getAlgorithm() }
/** Gets an input the algorithm is used on, for example the plain text input to be encrypted. */
DataFlow::Node getAnInput() { result = range.getAnInput() }
}
/** Provides classes for modeling new applications of a cryptographic algorithms. */
module CryptographicOperation {
/**
* A data-flow node that is an application of a cryptographic algorithm. For example,
* encryption, decryption, signature-validation.
*
* Extend this class to model new APIs. If you want to refine existing API models,
* extend `CryptographicOperation` instead.
*/
abstract class Range extends DataFlow::Node {
/** Gets the algorithm used, if it matches a known `CryptographicAlgorithm`. */
abstract CryptographicAlgorithm getAlgorithm();
/** Gets an input the algorithm is used on, for example the plain text input to be encrypted. */
abstract DataFlow::Node getAnInput();
}
}
}

34
python/ql/src/semmle/python/Constants.qll
View File

@@ -1,34 +0,0 @@
/** Standard builtin types and modules */
import python
/** the Python major version number */
int major_version() {
explicit_major_version(result)
or
not explicit_major_version(_) and
/* If there is more than one version, prefer 2 for backwards compatibilty */
(if py_flags_versioned("version.major", "2", "2") then result = 2 else result = 3)
}
/** the Python minor version number */
int minor_version() {
exists(string v | py_flags_versioned("version.minor", v, major_version().toString()) |
result = v.toInt()
)
}
/** the Python micro version number */
int micro_version() {
exists(string v | py_flags_versioned("version.micro", v, major_version().toString()) |
result = v.toInt()
)
}
private predicate explicit_major_version(int v) {
exists(string version | py_flags_versioned("language.version", version, _) |
version.charAt(0) = "2" and v = 2
or
version.charAt(0) = "3" and v = 3
)
}

741
python/ql/src/semmle/python/Exprs.qll
View File

@@ -1,741 +0,0 @@
import python
private import semmle.python.pointsto.PointsTo
private import semmle.python.objects.ObjectInternal
/** An expression */
class Expr extends Expr_, AstNode {
/** Gets the scope of this expression */
override Scope getScope() { py_scopes(this, result) }
/** Gets a textual representation of this element. */
override string toString() { result = "Expression" }
/** Gets the module in which this expression occurs */
Module getEnclosingModule() { result = this.getScope().getEnclosingModule() }
/**
* Whether this expression defines variable `v`
* If doing dataflow, then consider using SsaVariable.getDefinition() for more precision.
*/
predicate defines(Variable v) { this.getASubExpression+().defines(v) }
/** Whether this expression may have a side effect (as determined purely from its syntax) */
predicate hasSideEffects() {
/* If an exception raised by this expression handled, count that as a side effect */
this.getAFlowNode().getASuccessor().getNode() instanceof ExceptStmt
or
this.getASubExpression().hasSideEffects()
}
/** Whether this expression is a constant */
predicate isConstant() { not this.isVariable() }
/** Use isParenthesized instead. */
deprecated override predicate isParenthesised() { this.isParenthesized() }
/** Whether the parenthesized property of this expression is true. */
predicate isParenthesized() { Expr_.super.isParenthesised() }
private predicate isVariable() {
this.hasSideEffects()
or
this instanceof Name
or
exists(Expr e | e = this.getASubExpression() and e.isVariable())
}
override Location getLocation() { result = Expr_.super.getLocation() }
/** Gets an immediate (non-nested) sub-expression of this expression */
Expr getASubExpression() { none() }
/** Use StrConst.getText() instead */
deprecated string strValue() { none() }
override AstNode getAChildNode() { result = this.getASubExpression() }
/**
* NOTE: `refersTo` will be deprecated in 2019. Use `pointsTo` instead.
* Gets what this expression might "refer-to". Performs a combination of localized (intra-procedural) points-to
* analysis and global module-level analysis. This points-to analysis favours precision over recall. It is highly
* precise, but may not provide information for a significant number of flow-nodes.
* If the class is unimportant then use `refersTo(value)` or `refersTo(value, origin)` instead.
* NOTE: For complex dataflow, involving multiple stages of points-to analysis, it may be more precise to use
* `ControlFlowNode.refersTo(...)` instead.
*/
predicate refersTo(Object obj, ClassObject cls, AstNode origin) {
this.refersTo(_, obj, cls, origin)
}
/**
* NOTE: `refersTo` will be deprecated in 2019. Use `pointsTo` instead.
* Gets what this expression might "refer-to" in the given `context`.
*/
predicate refersTo(Context context, Object obj, ClassObject cls, AstNode origin) {
this.getAFlowNode().refersTo(context, obj, cls, origin.getAFlowNode())
}
/**
* NOTE: `refersTo` will be deprecated in 2019. Use `pointsTo` instead.
* Holds if this expression might "refer-to" to `value` which is from `origin`
* Unlike `this.refersTo(value, _, origin)`, this predicate includes results
* where the class cannot be inferred.
*/
pragma[nomagic]
predicate refersTo(Object obj, AstNode origin) {
this.getAFlowNode().refersTo(obj, origin.getAFlowNode())
}
/**
* NOTE: `refersTo` will be deprecated in 2019. Use `pointsTo` instead.
* Equivalent to `this.refersTo(value, _)`
*/
predicate refersTo(Object obj) { this.refersTo(obj, _) }
/**
* Holds if this expression might "point-to" to `value` which is from `origin`
* in the given `context`.
*/
predicate pointsTo(Context context, Value value, AstNode origin) {
this.getAFlowNode().pointsTo(context, value, origin.getAFlowNode())
}
/**
* Holds if this expression might "point-to" to `value` which is from `origin`.
*/
predicate pointsTo(Value value, AstNode origin) {
this.getAFlowNode().pointsTo(value, origin.getAFlowNode())
}
/**
* Holds if this expression might "point-to" to `value`.
*/
predicate pointsTo(Value value) { this.pointsTo(value, _) }
/** Gets a value that this expression might "point-to". */
Value pointsTo() { this.pointsTo(result) }
}
/** An assignment expression, such as `x := y` */
class AssignExpr extends AssignExpr_ {
override Expr getASubExpression() {
result = this.getValue() or
result = this.getTarget()
}
}
/** An attribute expression, such as `value.attr` */
class Attribute extends Attribute_ {
/* syntax: Expr.name */
override Expr getASubExpression() { result = this.getObject() }
override AttrNode getAFlowNode() { result = super.getAFlowNode() }
/** Gets the name of this attribute. That is the `name` in `obj.name` */
string getName() { result = Attribute_.super.getAttr() }
/** Gets the object of this attribute. That is the `obj` in `obj.name` */
Expr getObject() { result = Attribute_.super.getValue() }
/**
* Gets the expression corresponding to the object of the attribute, if the name of the attribute is `name`.
* Equivalent to `this.getObject() and this.getName() = name`.
*/
Expr getObject(string name) {
result = Attribute_.super.getValue() and
name = Attribute_.super.getAttr()
}
}
/** A subscript expression, such as `value[slice]` */
class Subscript extends Subscript_ {
/* syntax: Expr[Expr] */
override Expr getASubExpression() {
result = this.getIndex()
or
result = this.getObject()
}
Expr getObject() { result = Subscript_.super.getValue() }
override SubscriptNode getAFlowNode() { result = super.getAFlowNode() }
}
/** A call expression, such as `func(...)` */
class Call extends Call_ {
/* syntax: Expr(...) */
override Expr getASubExpression() {
result = this.getAPositionalArg() or
result = this.getAKeyword().getValue() or
result = this.getFunc()
}
override predicate hasSideEffects() { any() }
override string toString() { result = this.getFunc().toString() + "()" }
override CallNode getAFlowNode() { result = super.getAFlowNode() }
/** Gets a tuple (*) argument of this call. */
Expr getStarargs() { result = this.getAPositionalArg().(Starred).getValue() }
/** Gets a dictionary (**) argument of this call. */
Expr getKwargs() { result = this.getANamedArg().(DictUnpacking).getValue() }
/* Backwards compatibility */
/**
* Gets the nth keyword argument of this call expression, provided it is not preceded by a double-starred argument.
* This exists primarily for backwards compatibility. You are recommended to use
* Call.getNamedArg(index) instead.
*/
Keyword getKeyword(int index) {
result = this.getNamedArg(index) and
not exists(DictUnpacking d, int lower | d = this.getNamedArg(lower) and lower < index)
}
/**
* Gets a keyword argument of this call expression, provided it is not preceded by a double-starred argument.
* This exists primarily for backwards compatibility. You are recommended to use
* Call.getANamedArg() instead.
*/
Keyword getAKeyword() { result = this.getKeyword(_) }
/**
* Gets the positional argument at `index`, provided it is not preceded by a starred argument.
* This exists primarily for backwards compatibility. You are recommended to use
* Call.getPositionalArg(index) instead.
*/
Expr getArg(int index) {
result = this.getPositionalArg(index) and
not result instanceof Starred and
not exists(Starred s, int lower | s = this.getPositionalArg(lower) and lower < index)
}
/**
* Gets a positional argument, provided it is not preceded by a starred argument.
* This exists primarily for backwards compatibility. You are recommended to use
* Call.getAPositionalArg() instead.
*/
Expr getAnArg() { result = this.getArg(_) }
override AstNode getAChildNode() {
result = this.getAPositionalArg() or
result = this.getANamedArg() or
result = this.getFunc()
}
/** Gets the name of a named argument, including those passed in dict literals. */
string getANamedArgumentName() {
result = this.getAKeyword().getArg()
or
result = this.getKwargs().(Dict).getAKey().(StrConst).getText()
}
/** Gets the positional argument count of this call, provided there is no more than one tuple (*) argument. */
int getPositionalArgumentCount() {
count(this.getStarargs()) < 2 and
result = count(Expr arg | arg = this.getAPositionalArg() and not arg instanceof Starred)
}
/** Gets the tuple (*) argument of this call, provided there is exactly one. */
Expr getStarArg() {
count(this.getStarargs()) < 2 and
result = getStarargs()
}
}
/** A conditional expression such as, `body if test else orelse` */
class IfExp extends IfExp_ {
/* syntax: Expr if Expr else Expr */
override Expr getASubExpression() {
result = this.getTest() or result = this.getBody() or result = this.getOrelse()
}
override IfExprNode getAFlowNode() { result = super.getAFlowNode() }
}
/** A starred expression, such as the `*rest` in the assignment `first, *rest = seq` */
class Starred extends Starred_ {
/* syntax: *Expr */
override Expr getASubExpression() { result = this.getValue() }
}
/** A yield expression, such as `yield value` */
class Yield extends Yield_ {
/* syntax: yield Expr */
override Expr getASubExpression() { result = this.getValue() }
override predicate hasSideEffects() { any() }
}
/** A yield expression, such as `yield from value` */
class YieldFrom extends YieldFrom_ {
/* syntax: yield from Expr */
override Expr getASubExpression() { result = this.getValue() }
override predicate hasSideEffects() { any() }
}
/** A repr (backticks) expression, such as `` `value` `` */
class Repr extends Repr_ {
/* syntax: `Expr` */
override Expr getASubExpression() { result = this.getValue() }
override predicate hasSideEffects() { any() }
}
/* Constants */
/**
* A bytes constant, such as `b'ascii'`. Note that unadorned string constants such as
* `"hello"` are treated as Bytes for Python2, but Unicode for Python3.
*/
class Bytes extends StrConst {
/* syntax: b"hello" */
Bytes() { not this.isUnicode() }
override Object getLiteralObject() {
py_cobjecttypes(result, theBytesType()) and
py_cobjectnames(result, this.quotedString())
}
/**
* The extractor puts quotes into the name of each string (to prevent "0" clashing with 0).
* The following predicate help us match up a string/byte literals in the source
* which the equivalent object.
*/
private string quotedString() {
exists(string b_unquoted | b_unquoted = this.getS() | result = "b'" + b_unquoted + "'")
}
}
/** An ellipsis expression, such as `...` */
class Ellipsis extends Ellipsis_ {
/* syntax: ... */
override Expr getASubExpression() { none() }
}
/**
* Immutable literal expressions (except tuples).
* Consists of string (both unicode and byte) literals and numeric literals.
*/
abstract class ImmutableLiteral extends Expr {
abstract Object getLiteralObject();
abstract boolean booleanValue();
final Value getLiteralValue() { result.(ConstantObjectInternal).getLiteral() = this }
}
/** A numerical constant expression, such as `7` or `4.2` */
abstract class Num extends Num_, ImmutableLiteral {
override Expr getASubExpression() { none() }
/* We want to declare this abstract, but currently we cannot. */
override string toString() { result = "Num with missing toString" }
}
/** An integer numeric constant, such as `7` or `0x9` */
class IntegerLiteral extends Num {
/* syntax: 4 */
IntegerLiteral() { not this instanceof FloatLiteral and not this instanceof ImaginaryLiteral }
/**
* Gets the (integer) value of this constant. Will not return a result if the value does not fit into
* a 32 bit signed value
*/
int getValue() { result = this.getN().toInt() }
override string toString() { result = "IntegerLiteral" }
override Object getLiteralObject() {
py_cobjecttypes(result, theIntType()) and py_cobjectnames(result, this.getN())
or
py_cobjecttypes(result, theLongType()) and py_cobjectnames(result, this.getN())
}
override boolean booleanValue() {
this.getValue() = 0 and result = false
or
this.getValue() != 0 and result = true
}
}
/** A floating point numeric constant, such as `0.4` or `4e3` */
class FloatLiteral extends Num {
/* syntax: 4.2 */
FloatLiteral() {
not this instanceof ImaginaryLiteral and
this.getN().regexpMatch(".*[.eE].*")
}
float getValue() { result = this.getN().toFloat() }
override string toString() { result = "FloatLiteral" }
override Object getLiteralObject() {
py_cobjecttypes(result, theFloatType()) and py_cobjectnames(result, this.getN())
}
override boolean booleanValue() {
this.getValue() = 0.0 and result = false
or
// In QL 0.0 != -0.0
this.getValue() = -0.0 and result = false
or
this.getValue() != 0.0 and this.getValue() != -0.0 and result = true
}
}
/** An imaginary numeric constant, such as `3j` */
class ImaginaryLiteral extends Num {
private float value;
/* syntax: 1.0j */
ImaginaryLiteral() { value = this.getN().regexpCapture("(.+)j.*", 1).toFloat() }
/** Gets the value of this constant as a floating point value */
float getValue() { result = value }
override string toString() { result = "ImaginaryLiteral" }
override Object getLiteralObject() {
py_cobjecttypes(result, theComplexType()) and py_cobjectnames(result, this.getN())
}
override boolean booleanValue() {
this.getValue() = 0.0 and result = false
or
// In QL 0.0 != -0.0
this.getValue() = -0.0 and result = false
or
this.getValue() != 0.0 and this.getValue() != -0.0 and result = true
}
}
class NegativeIntegerLiteral extends ImmutableLiteral, UnaryExpr {
NegativeIntegerLiteral() {
this.getOp() instanceof USub and
this.getOperand() instanceof IntegerLiteral
}
override boolean booleanValue() { result = this.getOperand().(IntegerLiteral).booleanValue() }
override Object getLiteralObject() {
(py_cobjecttypes(result, theIntType()) or py_cobjecttypes(result, theLongType())) and
py_cobjectnames(result, "-" + this.getOperand().(IntegerLiteral).getN())
}
/**
* Gets the (integer) value of this constant. Will not return a result if the value does not fit into
* a 32 bit signed value
*/
int getValue() { result = -this.getOperand().(IntegerLiteral).getValue() }
}
/**
* A unicode string expression, such as `u"\u20ac"`. Note that unadorned string constants such as
* "hello" are treated as Bytes for Python2, but Unicode for Python3.
*/
class Unicode extends StrConst {
/* syntax: "hello" */
Unicode() { this.isUnicode() }
override Object getLiteralObject() {
py_cobjecttypes(result, theUnicodeType()) and
py_cobjectnames(result, this.quotedString())
}
/**
* The extractor puts quotes into the name of each string (to prevent "0" clashing with 0).
* The following predicate help us match up a string/byte literals in the source
* which the equivalent object.
*/
string quotedString() {
exists(string u_unquoted | u_unquoted = this.getS() | result = "u'" + u_unquoted + "'")
}
}
/* Compound Values */
/** A dictionary expression, such as `{'key':'value'}` */
class Dict extends Dict_ {
/* syntax: {Expr: Expr, ...} */
/** Gets the value of an item of this dict display */
Expr getAValue() { result = this.getAnItem().(DictDisplayItem).getValue() }
/**
* Gets the key of an item of this dict display, for those items that have keys
* E.g, in {'a':1, **b} this returns only 'a'
*/
Expr getAKey() { result = this.getAnItem().(KeyValuePair).getKey() }
override Expr getASubExpression() { result = this.getAValue() or result = this.getAKey() }
override AstNode getAChildNode() { result = this.getAnItem() }
}
/** A list expression, such as `[ 1, 3, 5, 7, 9 ]` */
class List extends List_ {
/* syntax: [Expr, ...] */
override Expr getASubExpression() { result = this.getAnElt() }
}
/** A set expression such as `{ 1, 3, 5, 7, 9 }` */
class Set extends Set_ {
/* syntax: {Expr, ...} */
override Expr getASubExpression() { result = this.getAnElt() }
}
class PlaceHolder extends PlaceHolder_ {
string getId() { result = this.getVariable().getId() }
override Expr getASubExpression() { none() }
override string toString() { result = "$" + this.getId() }
override NameNode getAFlowNode() { result = super.getAFlowNode() }
}
/** A tuple expression such as `( 1, 3, 5, 7, 9 )` */
class Tuple extends Tuple_ {
/* syntax: (Expr, ...) */
override Expr getASubExpression() { result = this.getAnElt() }
}
/**
* A (plain variable) name expression, such as `var`.
* `None`, `True` and `False` are excluded.
*/
class Name extends Name_ {
/* syntax: name */
string getId() { result = this.getVariable().getId() }
/** Whether this expression is a definition */
predicate isDefinition() {
py_expr_contexts(_, 5, this)
or
/* Treat Param as a definition (which it is) */
py_expr_contexts(_, 4, this)
or
/* The target in an augmented assignment is also a definition (and a use) */
exists(AugAssign aa | aa.getTarget() = this)
}
/**
* Whether this expression defines variable `v`
* If doing dataflow, then consider using SsaVariable.getDefinition() for more precision.
*/
override predicate defines(Variable v) {
this.isDefinition() and
v = this.getVariable()
}
/** Whether this expression is a deletion */
predicate isDeletion() { py_expr_contexts(_, 2, this) }
/**
* Whether this expression deletes variable `v`.
* If doing dataflow, then consider using SsaVariable.getDefinition() for more precision.
*/
predicate deletes(Variable v) {
this.isDeletion() and
v = this.getVariable()
}
/** Whether this expression is a use */
predicate isUse() { py_expr_contexts(_, 3, this) }
/**
* Whether this expression is a use of variable `v`
* If doing dataflow, then consider using SsaVariable.getAUse() for more precision.
*/
predicate uses(Variable v) {
this.isUse() and
v = this.getVariable()
}
override predicate isConstant() { none() }
override Expr getASubExpression() { none() }
override string toString() { result = this.getId() }
override NameNode getAFlowNode() { result = super.getAFlowNode() }
override predicate isArtificial() {
/* Artificial variable names in comprehensions all start with "." */
this.getId().charAt(0) = "."
}
}
class Filter extends Filter_ {
override Expr getASubExpression() {
result = this.getFilter()
or
result = this.getValue()
}
}
/** A slice. E.g `0:1` in the expression `x[0:1]` */
class Slice extends Slice_ {
override Expr getASubExpression() {
result = this.getStart() or
result = this.getStop() or
result = this.getStep()
}
}
/**
* Returns all string prefixes in the database that are explicitly marked as Unicode strings.
*
* Helper predicate for `StrConst::isUnicode`.
*/
pragma[nomagic]
private string unicode_prefix() {
result = any(Str_ s).getPrefix() and
result.charAt(_) in ["u", "U"]
}
/**
* Returns all string prefixes in the database that are _not_ explicitly marked as bytestrings.
*
* Helper predicate for `StrConst::isUnicode`.
*/
pragma[nomagic]
private string non_byte_prefix() {
result = any(Str_ s).getPrefix() and
not result.charAt(_) in ["b", "B"]
}
/** A string constant. */
class StrConst extends Str_, ImmutableLiteral {
/* syntax: "hello" */
predicate isUnicode() {
this.getPrefix() = unicode_prefix()
or
this.getPrefix() = non_byte_prefix() and
(
major_version() = 3
or
this.getEnclosingModule().hasFromFuture("unicode_literals")
)
}
deprecated override string strValue() { result = this.getS() }
override Expr getASubExpression() { none() }
override AstNode getAChildNode() { result = this.getAnImplicitlyConcatenatedPart() }
/** Gets the text of this str constant */
string getText() { result = this.getS() }
/** Whether this is a docstring */
predicate isDocString() { exists(Scope s | s.getDocString() = this) }
override boolean booleanValue() {
this.getText() = "" and result = false
or
this.getText() != "" and result = true
}
override Object getLiteralObject() { none() }
}
private predicate name_consts(Name_ n, string id) {
exists(Variable v | py_variables(v, n) and id = v.getId() |
id = "True" or id = "False" or id = "None"
)
}
/** A named constant, one of `None`, `True` or `False` */
abstract class NameConstant extends Name, ImmutableLiteral {
NameConstant() { name_consts(this, _) }
override Expr getASubExpression() { none() }
override string toString() { name_consts(this, result) }
override predicate isConstant() { any() }
override NameConstantNode getAFlowNode() { result = Name.super.getAFlowNode() }
override predicate isArtificial() { none() }
}
/** A boolean named constant, either `True` or `False` */
abstract class BooleanLiteral extends NameConstant { }
/** The boolean named constant `True` */
class True extends BooleanLiteral {
/* syntax: True */
True() { name_consts(this, "True") }
override Object getLiteralObject() { name_consts(this, "True") and result = theTrueObject() }
override boolean booleanValue() { result = true }
}
/** The boolean named constant `False` */
class False extends BooleanLiteral {
/* syntax: False */
False() { name_consts(this, "False") }
override Object getLiteralObject() { name_consts(this, "False") and result = theFalseObject() }
override boolean booleanValue() { result = false }
}
/** `None` */
class None extends NameConstant {
/* syntax: None */
None() { name_consts(this, "None") }
override Object getLiteralObject() { name_consts(this, "None") and result = theNoneObject() }
override boolean booleanValue() { result = false }
}
/** An await expression such as `await coro`. */
class Await extends Await_ {
/* syntax: await Expr */
override Expr getASubExpression() { result = this.getValue() }
}
/** A formatted string literal expression, such as `f'hello {world!s}'` */
class Fstring extends Fstring_ {
/* syntax: f"Yes!" */
override Expr getASubExpression() { result = this.getAValue() }
}
/**
* A formatted value (within a formatted string literal).
* For example, in the string `f'hello {world!s}'` the formatted value is `world!s`.
*/
class FormattedValue extends FormattedValue_ {
override Expr getASubExpression() {
result = this.getValue() or
result = this.getFormatSpec()
}
}
/* Expression Contexts */
/** A context in which an expression used */
class ExprContext extends ExprContext_ { }
/** Load context, the context of var in len(var) */
class Load extends Load_ { }
/** Store context, the context of var in var = 0 */
class Store extends Store_ { }
/** Delete context, the context of var in del var */
class Del extends Del_ { }
/** This is an artifact of the Python grammar which includes an AugLoad context, even though it is never used. */
library class AugLoad extends AugLoad_ { }
/** Augmented store context, the context of var in var += 1 */
class AugStore extends AugStore_ { }
/** Parameter context, the context of var in def f(var): pass */
class Param extends Param_ { }

523
python/ql/src/semmle/python/Files.qll
View File

@@ -1,523 +0,0 @@
import python
/** A file */
class File extends Container {
File() { files(this, _, _, _, _) }
/** DEPRECATED: Use `getAbsolutePath` instead. */
deprecated override string getName() { result = this.getAbsolutePath() }
/** DEPRECATED: Use `getAbsolutePath` instead. */
deprecated string getFullName() { result = this.getAbsolutePath() }
/**
* Holds if this element is at the specified location.
* The location spans column `startcolumn` of line `startline` to
* column `endcolumn` of line `endline` in file `filepath`.
* For more information, see
* [Locations](https://help.semmle.com/QL/learn-ql/ql/locations.html).
*/
predicate hasLocationInfo(
string filepath, int startline, int startcolumn, int endline, int endcolumn
) {
this.getAbsolutePath() = filepath and
startline = 0 and
startcolumn = 0 and
endline = 0 and
endcolumn = 0
}
/** Whether this file is a source code file. */
predicate fromSource() {
/* If we start to analyse .pyc files, then this will have to change. */
any()
}
/** Gets a short name for this file (just the file name) */
string getShortName() {
exists(string simple, string ext | files(this, _, simple, ext, _) | result = simple + ext)
}
private int lastLine() {
result = max(int i | exists(Location l | l.getFile() = this and l.getEndLine() = i))
}
/** Whether line n is empty (it contains neither code nor comment). */
predicate emptyLine(int n) {
n in [0 .. this.lastLine()] and
not occupied_line(this, n)
}
string getSpecifiedEncoding() {
exists(Comment c, Location l | l = c.getLocation() and l.getFile() = this |
l.getStartLine() < 3 and
result = c.getText().regexpCapture(".*coding[:=]\\s*([-\\w.]+).*", 1)
)
}
override string getAbsolutePath() { files(this, result, _, _, _) }
/** Gets the URL of this file. */
override string getURL() { result = "file://" + this.getAbsolutePath() + ":0:0:0:0" }
override Container getImportRoot(int n) {
/* File stem must be a legal Python identifier */
this.getStem().regexpMatch("[^\\d\\W]\\w*") and
result = this.getParent().getImportRoot(n)
}
/**
* Gets the contents of this file as a string.
* This will only work for those non-python files that
* are specified to be extracted.
*/
string getContents() { file_contents(this, result) }
/** Holds if this file is likely to get executed directly, and thus act as an entry point for execution. */
predicate isPossibleEntryPoint() {
// Only consider files in the source code, and not things like the standard library
exists(this.getRelativePath()) and
(
// The file doesn't have the extension `.py` but still contains Python statements
not this.getExtension().matches("py%") and
exists(Stmt s | s.getLocation().getFile() = this)
or
// The file contains the usual `if __name__ == '__main__':` construction
exists(If i, Name name, StrConst main, Cmpop op |
i.getScope().(Module).getFile() = this and
op instanceof Eq and
i.getTest().(Compare).compares(name, op, main) and
name.getId() = "__name__" and
main.getText() = "__main__"
) and
// Exclude files named `__main__.py`. These are often _not_ meant to be run directly, but
// contain this construct anyway.
//
// Their presence in a package (say, `foo`) means one can execute the package directly using
// `python -m foo` (which will run the `foo/__main__.py` file). Since being an entry point for
// execution means treating imports as absolute, this causes trouble, since when run with
// `python -m`, the interpreter uses the usual package semantics.
not this.getShortName() = "__main__.py"
or
// The file contains a `#!` line referencing the python interpreter
exists(Comment c |
c.getLocation().getFile() = this and
c.getLocation().getStartLine() = 1 and
c.getText().regexpMatch("^#! */.*python(2|3)?[ \\\\t]*$")
)
)
}
}
private predicate occupied_line(File f, int n) {
exists(Location l | l.getFile() = f |
l.getStartLine() = n
or
exists(StrConst s | s.getLocation() = l | n in [l.getStartLine() .. l.getEndLine()])
)
}
/** A folder (directory) */
class Folder extends Container {
Folder() { folders(this, _, _) }
/** DEPRECATED: Use `getAbsolutePath` instead. */
deprecated override string getName() { result = this.getAbsolutePath() }
/** DEPRECATED: Use `getBaseName` instead. */
deprecated string getSimple() { folders(this, _, result) }
/**
* Holds if this element is at the specified location.
* The location spans column `startcolumn` of line `startline` to
* column `endcolumn` of line `endline` in file `filepath`.
* For more information, see
* [Locations](https://help.semmle.com/QL/learn-ql/ql/locations.html).
*/
predicate hasLocationInfo(
string filepath, int startline, int startcolumn, int endline, int endcolumn
) {
this.getAbsolutePath() = filepath and
startline = 0 and
startcolumn = 0 and
endline = 0 and
endcolumn = 0
}
override string getAbsolutePath() { folders(this, result, _) }
/** Gets the URL of this folder. */
override string getURL() { result = "folder://" + this.getAbsolutePath() }
override Container getImportRoot(int n) {
this.isImportRoot(n) and result = this
or
/* Folder must be a legal Python identifier */
this.getBaseName().regexpMatch("[^\\d\\W]\\w*") and
result = this.getParent().getImportRoot(n)
}
}
/**
* A container is an abstract representation of a file system object that can
* hold elements of interest.
*/
abstract class Container extends @container {
Container getParent() { containerparent(result, this) }
/** Gets a child of this container */
deprecated Container getChild() { containerparent(this, result) }
/**
* Gets a textual representation of the path of this container.
*
* This is the absolute path of the container.
*/
string toString() { result = this.getAbsolutePath() }
/** Gets the name of this container */
abstract string getName();
/**
* Gets the relative path of this file or folder from the root folder of the
* analyzed source location. The relative path of the root folder itself is
* the empty string.
*
* This has no result if the container is outside the source root, that is,
* if the root folder is not a reflexive, transitive parent of this container.
*/
string getRelativePath() {
exists(string absPath, string pref |
absPath = this.getAbsolutePath() and sourceLocationPrefix(pref)
|
absPath = pref and result = ""
or
absPath = pref.regexpReplaceAll("/$", "") + "/" + result and
not result.matches("/%")
)
}
/** Whether this file or folder is part of the standard library */
predicate inStdlib() { this.inStdlib(_, _) }
/**
* Whether this file or folder is part of the standard library
* for version `major.minor`
*/
predicate inStdlib(int major, int minor) {
exists(Module m |
m.getPath() = this and
m.inStdLib(major, minor)
)
}
/* Standard cross-language API */
/** Gets a file or sub-folder in this container. */
Container getAChildContainer() { containerparent(this, result) }
/** Gets a file in this container. */
File getAFile() { result = this.getAChildContainer() }
/** Gets a sub-folder in this container. */
Folder getAFolder() { result = this.getAChildContainer() }
/**
* Gets the absolute, canonical path of this container, using forward slashes
* as path separator.
*
* The path starts with a _root prefix_ followed by zero or more _path
* segments_ separated by forward slashes.
*
* The root prefix is of one of the following forms:
*
* 1. A single forward slash `/` (Unix-style)
* 2. An upper-case drive letter followed by a colon and a forward slash,
* such as `C:/` (Windows-style)
* 3. Two forward slashes, a computer name, and then another forward slash,
* such as `//FileServer/` (UNC-style)
*
* Path segments are never empty (that is, absolute paths never contain two
* contiguous slashes, except as part of a UNC-style root prefix). Also, path
* segments never contain forward slashes, and no path segment is of the
* form `.` (one dot) or `..` (two dots).
*
* Note that an absolute path never ends with a forward slash, except if it is
* a bare root prefix, that is, the path has no path segments. A container
* whose absolute path has no segments is always a `Folder`, not a `File`.
*/
abstract string getAbsolutePath();
/**
* Gets the base name of this container including extension, that is, the last
* segment of its absolute path, or the empty string if it has no segments.
*
* Here are some examples of absolute paths and the corresponding base names
* (surrounded with quotes to avoid ambiguity):
*
* <table border="1">
* <tr><th>Absolute path</th><th>Base name</th></tr>
* <tr><td>"/tmp/tst.py"</td><td>"tst.py"</td></tr>
* <tr><td>"C:/Program Files (x86)"</td><td>"Program Files (x86)"</td></tr>
* <tr><td>"/"</td><td>""</td></tr>
* <tr><td>"C:/"</td><td>""</td></tr>
* <tr><td>"D:/"</td><td>""</td></tr>
* <tr><td>"//FileServer/"</td><td>""</td></tr>
* </table>
*/
string getBaseName() {
result = getAbsolutePath().regexpCapture(".*/(([^/]*?)(?:\\.([^.]*))?)", 1)
}
/**
* Gets the extension of this container, that is, the suffix of its base name
* after the last dot character, if any.
*
* In particular,
*
* - if the name does not include a dot, there is no extension, so this
* predicate has no result;
* - if the name ends in a dot, the extension is the empty string;
* - if the name contains multiple dots, the extension follows the last dot.
*
* Here are some examples of absolute paths and the corresponding extensions
* (surrounded with quotes to avoid ambiguity):
*
* <table border="1">
* <tr><th>Absolute path</th><th>Extension</th></tr>
* <tr><td>"/tmp/tst.py"</td><td>"py"</td></tr>
* <tr><td>"/tmp/.gitignore"</td><td>"gitignore"</td></tr>
* <tr><td>"/bin/bash"</td><td>not defined</td></tr>
* <tr><td>"/tmp/tst2."</td><td>""</td></tr>
* <tr><td>"/tmp/x.tar.gz"</td><td>"gz"</td></tr>
* </table>
*/
string getExtension() { result = getAbsolutePath().regexpCapture(".*/([^/]*?)(\\.([^.]*))?", 3) }
/**
* Gets the stem of this container, that is, the prefix of its base name up to
* (but not including) the last dot character if there is one, or the entire
* base name if there is not.
*
* Here are some examples of absolute paths and the corresponding stems
* (surrounded with quotes to avoid ambiguity):
*
* <table border="1">
* <tr><th>Absolute path</th><th>Stem</th></tr>
* <tr><td>"/tmp/tst.py"</td><td>"tst"</td></tr>
* <tr><td>"/tmp/.gitignore"</td><td>""</td></tr>
* <tr><td>"/bin/bash"</td><td>"bash"</td></tr>
* <tr><td>"/tmp/tst2."</td><td>"tst2"</td></tr>
* <tr><td>"/tmp/x.tar.gz"</td><td>"x.tar"</td></tr>
* </table>
*/
string getStem() { result = getAbsolutePath().regexpCapture(".*/([^/]*?)(?:\\.([^.]*))?", 1) }
File getFile(string baseName) {
result = this.getAFile() and
result.getBaseName() = baseName
}
Folder getFolder(string baseName) {
result = this.getAFolder() and
result.getBaseName() = baseName
}
Container getParentContainer() { this = result.getAChildContainer() }
Container getChildContainer(string baseName) {
result = this.getAChildContainer() and
result.getBaseName() = baseName
}
/**
* Gets a URL representing the location of this container.
*
* For more information see [Providing URLs](https://help.semmle.com/QL/learn-ql/ql/locations.html#providing-urls).
*/
abstract string getURL();
/** Holds if this folder is on the import path. */
predicate isImportRoot() { this.isImportRoot(_) }
/**
* Holds if this folder is on the import path, at index `n` in the list of
* paths. The list of paths is composed of the paths passed to the extractor and
* `sys.path`.
*/
predicate isImportRoot(int n) { this.getName() = import_path_element(n) }
/** Holds if this folder is the root folder for the standard library. */
predicate isStdLibRoot(int major, int minor) {
major = major_version() and
minor = minor_version() and
this.isStdLibRoot()
}
/** Holds if this folder is the root folder for the standard library. */
predicate isStdLibRoot() {
/*
* Look for a standard lib module and find its import path
* We use `os` as it is the most likely to be imported and
* `tty` because it is small for testing.
*/
exists(Module m | m.getName() = "os" or m.getName() = "tty" |
m.getFile().getImportRoot() = this
)
}
/** Gets the path element from which this container would be loaded. */
Container getImportRoot() {
exists(int n |
result = this.getImportRoot(n) and
not exists(int m |
exists(this.getImportRoot(m)) and
m < n
)
)
}
/** Gets the path element from which this container would be loaded, given the index into the list of possible paths `n`. */
abstract Container getImportRoot(int n);
}
private string import_path_element(int n) {
exists(string path, string pathsep, int k |
path = get_path("extractor.path") and k = 0
or
path = get_path("sys.path") and k = count(get_path("extractor.path").splitAt(pathsep))
|
py_flags_versioned("os.pathsep", pathsep, _) and
result = path.splitAt(pathsep, n - k).replaceAll("\\", "/")
)
}
private string get_path(string name) { py_flags_versioned(name, result, _) }
class Location extends @location {
/** Gets the file for this location */
File getFile() { result = this.getPath() }
private Container getPath() {
locations_default(this, result, _, _, _, _)
or
exists(Module m | locations_ast(this, m, _, _, _, _) | result = m.getPath())
}
/** Gets the 1-based line number (inclusive) where this location starts. */
int getStartLine() {
locations_default(this, _, result, _, _, _) or
locations_ast(this, _, result, _, _, _)
}
/** Gets the 1-based column number (inclusive) where this location starts. */
int getStartColumn() {
locations_default(this, _, _, result, _, _) or
locations_ast(this, _, _, result, _, _)
}
/** Gets the 1-based line number (inclusive) where this location ends. */
int getEndLine() {
locations_default(this, _, _, _, result, _) or
locations_ast(this, _, _, _, result, _)
}
/** Gets the 1-based column number (inclusive) where this location ends. */
int getEndColumn() {
locations_default(this, _, _, _, _, result) or
locations_ast(this, _, _, _, _, result)
}
/** Gets a textual representation of this element. */
string toString() {
result = this.getPath().getAbsolutePath() + ":" + this.getStartLine().toString()
}
/**
* Holds if this element is at the specified location.
* The location spans column `startcolumn` of line `startline` to
* column `endcolumn` of line `endline` in file `filepath`.
* For more information, see
* [Locations](https://help.semmle.com/QL/learn-ql/ql/locations.html).
*/
predicate hasLocationInfo(
string filepath, int startline, int startcolumn, int endline, int endcolumn
) {
exists(File f | f.getAbsolutePath() = filepath |
locations_default(this, f, startline, startcolumn, endline, endcolumn)
or
exists(Module m | m.getFile() = f |
locations_ast(this, m, startline, startcolumn, endline, endcolumn)
)
)
or
// Packages have no suitable filepath, so we use just the path instead.
exists(Module m | not exists(m.getFile()) |
filepath = m.getPath().getAbsolutePath() and
locations_ast(this, m, startline, startcolumn, endline, endcolumn)
)
}
}
/** A non-empty line in the source code */
class Line extends @py_line {
/**
* Holds if this element is at the specified location.
* The location spans column `startcolumn` of line `startline` to
* column `endcolumn` of line `endline` in file `filepath`.
* For more information, see
* [Locations](https://help.semmle.com/QL/learn-ql/ql/locations.html).
*/
predicate hasLocationInfo(
string filepath, int startline, int startcolumn, int endline, int endcolumn
) {
exists(Module m |
m.getFile().getAbsolutePath() = filepath and
endline = startline and
startcolumn = 1 and
py_line_lengths(this, m, startline, endcolumn)
)
}
/** Gets a textual representation of this element. */
string toString() {
exists(Module m | py_line_lengths(this, m, _, _) |
result = m.getFile().getShortName() + ":" + this.getLineNumber().toString()
)
}
/** Gets the line number of this line */
int getLineNumber() { py_line_lengths(this, _, result, _) }
/** Gets the length of this line */
int getLength() { py_line_lengths(this, _, _, result) }
/** Gets the file for this line */
Module getModule() { py_line_lengths(this, result, _, _) }
}
/**
* A syntax error. Note that if there is a syntax error in a module,
* much information about that module will be lost
*/
class SyntaxError extends Location {
SyntaxError() { py_syntax_error_versioned(this, _, major_version().toString()) }
override string toString() { result = "Syntax Error" }
/** Gets the message corresponding to this syntax error */
string getMessage() { py_syntax_error_versioned(this, result, major_version().toString()) }
}
/**
* An encoding error. Note that if there is an encoding error in a module,
* much information about that module will be lost
*/
class EncodingError extends SyntaxError {
EncodingError() {
/* Leave spaces around 'decode' in unlikely event it occurs as a name in a syntax error */
this.getMessage().toLowerCase().matches("% decode %")
}
override string toString() { result = "Encoding Error" }
}

1217
python/ql/src/semmle/python/Flow.qll
View File

File diff suppressed because it is too large Load Diff

32
python/ql/src/semmle/python/Frameworks.qll
View File

@@ -1,32 +0,0 @@
/**
* Helper file that imports all framework modeling.
*/
// If you add modeling of a new framework/library, remember to add it it to the docs in
// `docs/codeql/support/reusables/frameworks.rst`
private import semmle.python.frameworks.Aioch
private import semmle.python.frameworks.Aiohttp
private import semmle.python.frameworks.ClickhouseDriver
private import semmle.python.frameworks.Cryptodome
private import semmle.python.frameworks.Cryptography
private import semmle.python.frameworks.Dill
private import semmle.python.frameworks.Django
private import semmle.python.frameworks.Fabric
private import semmle.python.frameworks.Flask
private import semmle.python.frameworks.Idna
private import semmle.python.frameworks.Invoke
private import semmle.python.frameworks.Jmespath
private import semmle.python.frameworks.MarkupSafe
private import semmle.python.frameworks.Multidict
private import semmle.python.frameworks.Mysql
private import semmle.python.frameworks.MySQLdb
private import semmle.python.frameworks.Psycopg2
private import semmle.python.frameworks.PyMySQL
private import semmle.python.frameworks.Rsa
private import semmle.python.frameworks.Simplejson
private import semmle.python.frameworks.Stdlib
private import semmle.python.frameworks.Tornado
private import semmle.python.frameworks.Twisted
private import semmle.python.frameworks.Ujson
private import semmle.python.frameworks.Yaml
private import semmle.python.frameworks.Yarl

380
python/ql/src/semmle/python/Function.qll
View File

@@ -1,380 +0,0 @@
import python
/**
* A function, independent of defaults and binding.
* It is the syntactic entity that is compiled to a code object.
*/
class Function extends Function_, Scope, AstNode {
/** The expression defining this function */
CallableExpr getDefinition() { result = this.getParent() }
/**
* The scope in which this function occurs, will be a class for a method,
* another function for nested functions, generator expressions or comprehensions,
* or a module for a plain function.
*/
override Scope getEnclosingScope() { result = this.getParent().(Expr).getScope() }
override Scope getScope() { result = this.getEnclosingScope() }
/** Whether this function is declared in a class */
predicate isMethod() { exists(Class cls | this.getEnclosingScope() = cls) }
/** Whether this is a special method, that is does its name have the form `__xxx__` (except `__init__`) */
predicate isSpecialMethod() {
this.isMethod() and
exists(string name | this.getName() = name |
name.matches("\\_\\_%\\_\\_") and
name != "__init__"
)
}
/**
* Whether this function is a generator function,
* that is whether it contains a yield or yield-from expression
*/
predicate isGenerator() {
exists(Yield y | y.getScope() = this)
or
exists(YieldFrom y | y.getScope() = this)
}
/**
* Holds if this function represents a lambda.
*
* The extractor reifies each lambda expression as a (local) function with the name
* "lambda". As `lambda` is a keyword in Python, it's impossible to create a function with this
* name otherwise, and so it's impossible to get a non-lambda function accidentally
* classified as a lambda.
*/
predicate isLambda() { this.getName() = "lambda" }
/** Whether this function is declared in a class and is named `__init__` */
predicate isInitMethod() { this.isMethod() and this.getName() = "__init__" }
/** Gets a decorator of this function */
Expr getADecorator() { result = this.getDefinition().(FunctionExpr).getADecorator() }
/** Gets the name of the nth argument (for simple arguments) */
string getArgName(int index) { result = this.getArg(index).(Name).getId() }
Parameter getArgByName(string name) {
(
result = this.getAnArg()
or
result = this.getAKeywordOnlyArg()
) and
result.(Name).getId() = name
}
override Location getLocation() { py_scope_location(result, this) }
override string toString() { result = "Function " + this.getName() }
/** Gets the statements forming the body of this function */
override StmtList getBody() { result = Function_.super.getBody() }
/** Gets the nth statement in the function */
override Stmt getStmt(int index) { result = Function_.super.getStmt(index) }
/** Gets a statement in the function */
override Stmt getAStmt() { result = Function_.super.getAStmt() }
/** Gets the name used to define this function */
override string getName() { result = Function_.super.getName() }
/** Gets the metrics for this function */
FunctionMetrics getMetrics() { result = this }
/** Gets the FunctionObject corresponding to this function */
FunctionObject getFunctionObject() { result.getOrigin() = this.getDefinition() }
/**
* Whether this function is a procedure, that is, it has no explicit return statement and always returns None.
* Note that generator and async functions are not procedures as they return generators and coroutines respectively.
*/
predicate isProcedure() {
not exists(this.getReturnNode()) and
exists(this.getFallthroughNode()) and
not this.isGenerator() and
not this.isAsync()
}
/** Gets the number of positional parameters */
int getPositionalParameterCount() { result = count(this.getAnArg()) }
/** Gets the number of keyword-only parameters */
int getKeywordOnlyParameterCount() { result = count(this.getAKeywordOnlyArg()) }
/** Whether this function accepts a variable number of arguments. That is, whether it has a starred (*arg) parameter. */
predicate hasVarArg() { exists(this.getVararg()) }
/** Whether this function accepts arbitrary keyword arguments. That is, whether it has a double-starred (**kwarg) parameter. */
predicate hasKwArg() { exists(this.getKwarg()) }
override AstNode getAChildNode() {
result = this.getAStmt() or
result = this.getAnArg() or
result = this.getVararg() or
result = this.getAKeywordOnlyArg() or
result = this.getKwarg()
}
/**
* Gets the qualified name for this function.
* Should return the same name as the `__qualname__` attribute on functions in Python 3.
*/
string getQualifiedName() {
this.getEnclosingScope() instanceof Module and result = this.getName()
or
exists(string enclosing_name |
enclosing_name = this.getEnclosingScope().(Function).getQualifiedName()
or
enclosing_name = this.getEnclosingScope().(Class).getQualifiedName()
|
result = enclosing_name + "." + this.getName()
)
}
/** Gets the nth keyword-only parameter of this function. */
Name getKeywordOnlyArg(int n) { result = Function_.super.getKwonlyarg(n) }
/** Gets a keyword-only parameter of this function. */
Name getAKeywordOnlyArg() { result = this.getKeywordOnlyArg(_) }
override Scope getEvaluatingScope() {
major_version() = 2 and
exists(Comp comp | comp.getFunction() = this | result = comp.getEvaluatingScope())
or
not exists(Comp comp | comp.getFunction() = this) and result = this
or
major_version() = 3 and result = this
}
override predicate containsInScope(AstNode inner) { Scope.super.containsInScope(inner) }
override predicate contains(AstNode inner) { Scope.super.contains(inner) }
/** Gets a control flow node for a return value of this function */
ControlFlowNode getAReturnValueFlowNode() {
exists(Return ret |
ret.getScope() = this and
ret.getValue() = result.getNode()
)
}
}
/** A def statement. Note that FunctionDef extends Assign as a function definition binds the newly created function */
class FunctionDef extends Assign {
/* syntax: def name(...): ... */
FunctionDef() {
/* This is an artificial assignment the rhs of which is a (possibly decorated) FunctionExpr */
exists(FunctionExpr f | this.getValue() = f or this.getValue() = f.getADecoratorCall())
}
override string toString() { result = "FunctionDef" }
/** Gets the function for this statement */
Function getDefinedFunction() {
exists(FunctionExpr func | this.containsInScope(func) and result = func.getInnerScope())
}
override Stmt getLastStatement() { result = this.getDefinedFunction().getLastStatement() }
}
class FastLocalsFunction extends Function {
/** A function that uses 'fast' locals, stored in the frame not in a dictionary. */
FastLocalsFunction() {
not exists(ImportStar i | i.getScope() = this) and
not exists(Exec e | e.getScope() = this)
}
}
/** A parameter. Either a Tuple or a Name (always a Name for Python 3) */
class Parameter extends Parameter_ {
Parameter() {
/* Parameter_ is just defined as a Name or Tuple, narrow to actual parameters */
exists(ParameterList pl | py_exprs(this, _, pl, _))
or
exists(Function f |
f.getVararg() = this
or
f.getKwarg() = this
or
f.getAKeywordOnlyArg() = this
)
}
Location getLocation() {
result = this.asName().getLocation()
or
result = this.asTuple().getLocation()
}
/** Gets this parameter if it is a Name (not a Tuple) */
Name asName() { result = this }
/** Gets this parameter if it is a Tuple (not a Name) */
Tuple asTuple() { result = this }
/** Gets the expression for the default value of this parameter */
Expr getDefault() {
exists(Function f, int i, Arguments args | args = f.getDefinition().getArgs() |
// positional (normal)
f.getArg(i) = this and
result = args.getDefault(i)
)
or
exists(Function f, int i, Arguments args | args = f.getDefinition().getArgs() |
// keyword-only
f.getKeywordOnlyArg(i) = this and
result = args.getKwDefault(i)
)
}
/** Gets the annotation expression of this parameter */
Expr getAnnotation() {
exists(Function f, int i, Arguments args | args = f.getDefinition().getArgs() |
// positional (normal)
f.getArg(i) = this and
result = args.getAnnotation(i)
)
or
exists(Function f, int i, Arguments args | args = f.getDefinition().getArgs() |
// keyword-only
f.getKeywordOnlyArg(i) = this and
result = args.getKwAnnotation(i)
)
or
exists(Function f, Arguments args | args = f.getDefinition().getArgs() |
f.getKwarg() = this and
result = args.getKwargannotation()
or
f.getVararg() = this and
result = args.getVarargannotation()
)
}
Variable getVariable() { result.getAnAccess() = this.asName() }
/**
* Gets the position of this parameter (if any).
* No result if this is a "varargs", "kwargs", or keyword-only parameter.
*/
int getPosition() { exists(Function f | f.getArg(result) = this) }
/** Gets the name of this parameter */
string getName() { result = this.asName().getId() }
/** Holds if this parameter is the first parameter of a method. It is not necessarily called "self" */
predicate isSelf() {
exists(Function f |
f.getArg(0) = this and
f.isMethod()
)
}
/**
* Holds if this parameter is a "varargs" parameter.
* The `varargs` in `f(a, b, *varargs)`.
*/
predicate isVarargs() { exists(Function func | func.getVararg() = this) }
/**
* Holds if this parameter is a "kwargs" parameter.
* The `kwargs` in `f(a, b, **kwargs)`.
*/
predicate isKwargs() { exists(Function func | func.getKwarg() = this) }
}
/** An expression that generates a callable object, either a function expression or a lambda */
abstract class CallableExpr extends Expr {
/**
* Gets The default values and annotations (type-hints) for the arguments of this callable.
*
* This predicate is called getArgs(), rather than getParameters() for compatibility with Python's AST module.
*/
abstract Arguments getArgs();
/** Gets the function scope of this code expression. */
abstract Function getInnerScope();
}
/** An (artificial) expression corresponding to a function definition. */
class FunctionExpr extends FunctionExpr_, CallableExpr {
override Expr getASubExpression() {
result = this.getArgs().getASubExpression() or
result = this.getReturns()
}
override predicate hasSideEffects() { any() }
Call getADecoratorCall() {
result.getArg(0) = this or
result.getArg(0) = this.getADecoratorCall()
}
/** Gets a decorator of this function expression */
Expr getADecorator() { result = this.getADecoratorCall().getFunc() }
override AstNode getAChildNode() {
result = this.getASubExpression()
or
result = this.getInnerScope()
}
override Function getInnerScope() { result = FunctionExpr_.super.getInnerScope() }
override Arguments getArgs() { result = FunctionExpr_.super.getArgs() }
}
/** A lambda expression, such as `lambda x: x+1` */
class Lambda extends Lambda_, CallableExpr {
/** Gets the expression to the right of the colon in this lambda expression */
Expr getExpression() {
exists(Return ret | ret = this.getInnerScope().getStmt(0) | result = ret.getValue())
}
override Expr getASubExpression() { result = this.getArgs().getASubExpression() }
override AstNode getAChildNode() {
result = this.getASubExpression() or
result = this.getInnerScope()
}
override Function getInnerScope() { result = Lambda_.super.getInnerScope() }
override Arguments getArgs() { result = Lambda_.super.getArgs() }
}
/**
* The default values and annotations (type hints) for the arguments in a function definition.
*
* Annotations (PEP 3107) is a general mechanism for providing annotations for a function,
* that is generally only used for type hints today (PEP 484).
*/
class Arguments extends Arguments_ {
Expr getASubExpression() {
result = this.getADefault() or
result = this.getAKwDefault() or
//
result = this.getAnAnnotation() or
result = this.getVarargannotation() or
result = this.getAKwAnnotation() or
result = this.getKwargannotation()
}
// The following 4 methods are overwritten to provide better QLdoc. Since the
// Arguments_ is auto-generated, we can't change the poor auto-generated docs there :(
/** Gets the default value for the `index`'th positional parameter. */
override Expr getDefault(int index) { result = super.getDefault(index) }
/** Gets the default value for the `index`'th keyword-only parameter. */
override Expr getKwDefault(int index) { result = super.getKwDefault(index) }
/** Gets the annotation for the `index`'th positional parameter. */
override Expr getAnnotation(int index) { result = super.getAnnotation(index) }
/** Gets the annotation for the `index`'th keyword-only parameter. */
override Expr getKwAnnotation(int index) { result = super.getKwAnnotation(index) }
}

68
python/ql/src/semmle/python/GuardedControlFlow.qll
View File

@@ -1,68 +0,0 @@
import python
/** A basic block which terminates in a condition, splitting the subsequent control flow */
class ConditionBlock extends BasicBlock {
ConditionBlock() {
exists(ControlFlowNode succ |
succ = this.getATrueSuccessor() or succ = this.getAFalseSuccessor()
)
}
/** Basic blocks controlled by this condition, i.e. those BBs for which the condition is testIsTrue */
predicate controls(BasicBlock controlled, boolean testIsTrue) {
/*
* For this block to control the block 'controlled' with 'testIsTrue' the following must be true:
* Execution must have passed through the test i.e. 'this' must strictly dominate 'controlled'.
* Execution must have passed through the 'testIsTrue' edge leaving 'this'.
*
* Although "passed through the true edge" implies that this.getATrueSuccessor() dominates 'controlled',
* the reverse is not true, as flow may have passed through another edge to get to this.getATrueSuccessor()
* so we need to assert that this.getATrueSuccessor() dominates 'controlled' *and* that
* all predecessors of this.getATrueSuccessor() are either this or dominated by this.getATrueSuccessor().
*
* For example, in the following python snippet:
* <code>
* if x:
* controlled
* false_successor
* uncontrolled
* </code>
* false_successor dominates uncontrolled, but not all of its predecessors are this (if x)
* or dominated by itself. Whereas in the following code:
* <code>
* if x:
* while controlled:
* also_controlled
* false_successor
* uncontrolled
* </code>
* the block 'while controlled' is controlled because all of its predecessors are this (if x)
* or (in the case of 'also_controlled') dominated by itself.
*
* The additional constraint on the predecessors of the test successor implies
* that `this` strictly dominates `controlled` so that isn't necessary to check
* directly.
*/
exists(BasicBlock succ |
testIsTrue = true and succ = this.getATrueSuccessor()
or
testIsTrue = false and succ = this.getAFalseSuccessor()
|
succ.dominates(controlled) and
forall(BasicBlock pred | pred.getASuccessor() = succ | pred = this or succ.dominates(pred))
)
}
/** Holds if this condition controls the edge `pred->succ`, i.e. those edges for which the condition is `testIsTrue`. */
predicate controlsEdge(BasicBlock pred, BasicBlock succ, boolean testIsTrue) {
this.controls(pred, testIsTrue) and succ = pred.getASuccessor()
or
pred = this and
(
testIsTrue = true and succ = this.getATrueSuccessor()
or
testIsTrue = false and succ = this.getAFalseSuccessor()
)
}
}

251
python/ql/src/semmle/python/Import.qll
View File

@@ -1,251 +0,0 @@
import python
private import semmle.python.types.Builtins
/**
* An alias in an import statement, the `mod as name` part of `import mod as name`. May be artificial;
* `import x` is transformed into `import x as x`
*/
class Alias extends Alias_ {
Location getLocation() { result = this.getValue().getLocation() }
}
private predicate valid_module_name(string name) {
exists(Module m | m.getName() = name)
or
exists(Builtin cmod | cmod.getClass() = Builtin::special("ModuleType") and cmod.getName() = name)
}
/** An artificial expression representing an import */
class ImportExpr extends ImportExpr_ {
private string basePackageName(int n) {
n = 1 and result = this.getEnclosingModule().getPackageName()
or
exists(string bpnm1 |
bpnm1 = this.basePackageName(n - 1) and
bpnm1.matches("%.%") and
result = bpnm1.regexpReplaceAll("\\.[^.]*$", "")
)
}
private predicate implicitRelativeImportsAllowed() {
// relative imports are no longer allowed in Python 3
major_version() < 3 and
// and can be explicitly turned off in later versions of Python 2
not getEnclosingModule().hasFromFuture("absolute_import")
}
/**
* The language specifies level as -1 if relative imports are to be tried first, 0 for absolute imports,
* and level > 0 for explicit relative imports.
*/
override int getLevel() {
exists(int l | l = super.getLevel() |
l > 0 and result = l
or
/* The extractor may set level to 0 even though relative imports apply */
l = 0 and
(if this.implicitRelativeImportsAllowed() then result = -1 else result = 0)
)
}
/**
* If this import is relative, and relative imports are allowed, compute
* the name of the topmost module that will be imported.
*/
private string relativeTopName() {
getLevel() = -1 and
result = basePackageName(1) + "." + this.getTopName() and
valid_module_name(result)
}
private string qualifiedTopName() {
if this.getLevel() <= 0
then result = this.getTopName()
else (
result = basePackageName(this.getLevel()) and
valid_module_name(result)
)
}
/**
* Gets the name by which the lowest level module or package is imported.
* NOTE: This is the name that used to import the module,
* which may not be the name of the module.
*/
string bottomModuleName() {
result = relativeTopName() + this.remainderOfName()
or
not exists(relativeTopName()) and
result = this.qualifiedTopName() + this.remainderOfName()
}
/** Gets the name of topmost module or package being imported */
string topModuleName() {
result = relativeTopName()
or
not exists(relativeTopName()) and
result = this.qualifiedTopName()
}
/**
* Gets the full name of the module resulting from evaluating this import.
* NOTE: This is the name that used to import the module,
* which may not be the name of the module.
*/
string getImportedModuleName() {
exists(string bottomName | bottomName = this.bottomModuleName() |
if this.isTop() then result = topModuleName() else result = bottomName
)
}
/**
* Gets the names of the modules that may be imported by this import.
* For example this predicate would return 'x' and 'x.y' for `import x.y`
*/
string getAnImportedModuleName() {
result = this.bottomModuleName()
or
result = this.getAnImportedModuleName().regexpReplaceAll("\\.[^.]*$", "")
}
override Expr getASubExpression() { none() }
override predicate hasSideEffects() { any() }
private string getTopName() { result = this.getName().regexpReplaceAll("\\..*", "") }
private string remainderOfName() {
not exists(this.getName()) and result = ""
or
this.getLevel() <= 0 and result = this.getName().regexpReplaceAll("^[^\\.]*", "")
or
this.getLevel() > 0 and result = "." + this.getName()
}
/**
* Whether this import is relative, that is not absolute.
* See https://www.python.org/dev/peps/pep-0328/
*/
predicate isRelative() {
/* Implicit */
exists(this.relativeTopName())
or
/* Explicit */
this.getLevel() > 0
}
}
/** A `from ... import ...` expression */
class ImportMember extends ImportMember_ {
override Expr getASubExpression() { result = this.getModule() }
override predicate hasSideEffects() {
/* Strictly this only has side-effects if the module is a package */
any()
}
/**
* Gets the full name of the module resulting from evaluating this import.
* NOTE: This is the name that used to import the module,
* which may not be the name of the module.
*/
string getImportedModuleName() {
result = this.getModule().(ImportExpr).getImportedModuleName() + "." + this.getName()
}
override ImportMemberNode getAFlowNode() { result = super.getAFlowNode() }
}
/** An import statement */
class Import extends Import_ {
/* syntax: import modname */
private ImportExpr getAModuleExpr() {
result = this.getAName().getValue()
or
result = this.getAName().getValue().(ImportMember).getModule()
}
/**
* Use getAnImportedModuleName(),
* possibly combined with ModuleObject.importedAs()
* Gets a module imported by this import statement
*/
deprecated Module getAModule() { result.getName() = this.getAnImportedModuleName() }
/** Whether this a `from ... import ...` statement */
predicate isFromImport() { this.getAName().getValue() instanceof ImportMember }
override Expr getASubExpression() {
result = this.getAModuleExpr() or
result = this.getAName().getAsname() or
result = this.getAName().getValue()
}
override Stmt getASubStatement() { none() }
/**
* Gets the name of an imported module.
* For example, for the import statement `import bar` which
* is a relative import in package "foo", this would return
* "foo.bar".
* The import statment `from foo import bar` would return
* `foo` and `foo.bar`
*/
string getAnImportedModuleName() {
result = this.getAModuleExpr().getAnImportedModuleName()
or
exists(ImportMember m, string modname |
m = this.getAName().getValue() and
modname = m.getModule().(ImportExpr).getImportedModuleName()
|
result = modname
or
result = modname + "." + m.getName()
)
}
}
/** An import * statement */
class ImportStar extends ImportStar_ {
/* syntax: from modname import * */
ImportExpr getModuleExpr() {
result = this.getModule()
or
result = this.getModule().(ImportMember).getModule()
}
override string toString() { result = "from " + this.getModuleExpr().getName() + " import *" }
/**
* Use getAnImportedModuleName(),
* possibly combined with ModuleObject.importedAs()
* Gets the module imported by this import * statement
*/
deprecated Module getTheModule() { result.getName() = this.getImportedModuleName() }
override Expr getASubExpression() { result = this.getModule() }
override Stmt getASubStatement() { none() }
/** Gets the name of the imported module. */
string getImportedModuleName() { result = this.getModuleExpr().getImportedModuleName() }
}
/**
* A statement that imports a module. This can be any statement that includes the `import` keyword,
* such as `import sys`, `from sys import version` or `from sys import *`.
*/
class ImportingStmt extends Stmt {
ImportingStmt() {
this instanceof Import
or
this instanceof ImportStar
}
/** Gets the name of an imported module. */
string getAnImportedModuleName() {
result = this.(Import).getAnImportedModuleName()
or
result = this.(ImportStar).getImportedModuleName()
}
}

60
python/ql/src/semmle/python/Keywords.qll
View File

@@ -1,60 +0,0 @@
import python
class KeyValuePair extends KeyValuePair_, DictDisplayItem {
/* syntax: Expr : Expr */
override Location getLocation() { result = KeyValuePair_.super.getLocation() }
override string toString() { result = KeyValuePair_.super.toString() }
/** Gets the value of this dictionary unpacking. */
override Expr getValue() { result = KeyValuePair_.super.getValue() }
override Scope getScope() { result = this.getValue().getScope() }
override AstNode getAChildNode() {
result = this.getKey()
or
result = this.getValue()
}
}
/** A double-starred expression in a call or dict literal. */
class DictUnpacking extends DictUnpacking_, DictUnpackingOrKeyword, DictDisplayItem {
override Location getLocation() { result = DictUnpacking_.super.getLocation() }
override string toString() { result = DictUnpacking_.super.toString() }
/** Gets the value of this dictionary unpacking. */
override Expr getValue() { result = DictUnpacking_.super.getValue() }
override Scope getScope() { result = this.getValue().getScope() }
override AstNode getAChildNode() { result = this.getValue() }
}
abstract class DictUnpackingOrKeyword extends DictItem {
abstract Expr getValue();
override string toString() { result = "DictUnpackingOrKeyword with missing toString" }
}
abstract class DictDisplayItem extends DictItem {
abstract Expr getValue();
override string toString() { result = "DictDisplayItem with missing toString" }
}
/** A keyword argument in a call. For example `arg=expr` in `foo(0, arg=expr)` */
class Keyword extends Keyword_, DictUnpackingOrKeyword {
/* syntax: name = Expr */
override Location getLocation() { result = Keyword_.super.getLocation() }
override string toString() { result = Keyword_.super.toString() }
/** Gets the value of this keyword argument. */
override Expr getValue() { result = Keyword_.super.getValue() }
override Scope getScope() { result = this.getValue().getScope() }
override AstNode getAChildNode() { result = this.getValue() }
}

333
python/ql/src/semmle/python/Metrics.qll
View File

@@ -1,333 +0,0 @@
import python
/** The metrics for a function */
class FunctionMetrics extends Function {
/**
* Gets the total number of lines (including blank lines)
* from the definition to the end of the function
*/
int getNumberOfLines() { py_alllines(this, result) }
/** Gets the number of lines of code in the function */
int getNumberOfLinesOfCode() { py_codelines(this, result) }
/** Gets the number of lines of comments in the function */
int getNumberOfLinesOfComments() { py_commentlines(this, result) }
/** Gets the number of lines of docstring in the function */
int getNumberOfLinesOfDocStrings() { py_docstringlines(this, result) }
/**
* Cyclomatic complexity:
* The number of linearly independent paths through the source code.
* Computed as E - N + 2P,
* where
* E = the number of edges of the graph.
* N = the number of nodes of the graph.
* P = the number of connected components, which for a single function is 1.
*/
int getCyclomaticComplexity() {
exists(int E, int N |
N = count(BasicBlock b | b = this.getABasicBlock() and b.likelyReachable()) and
E =
count(BasicBlock b1, BasicBlock b2 |
b1 = this.getABasicBlock() and
b1.likelyReachable() and
b2 = this.getABasicBlock() and
b2.likelyReachable() and
b2 = b1.getASuccessor() and
not b1.unlikelySuccessor(b2)
)
|
result = E - N + 2
)
}
private BasicBlock getABasicBlock() {
result = this.getEntryNode().getBasicBlock()
or
exists(BasicBlock mid | mid = this.getABasicBlock() and result = mid.getASuccessor())
}
/**
* Dependency of Callables
* One callable "this" depends on another callable "result"
* if "this" makes some call to a method that may end up being "result".
*/
FunctionMetrics getADependency() {
result != this and
not non_coupling_method(result) and
exists(Call call | call.getScope() = this |
exists(FunctionObject callee | callee.getFunction() = result |
call.getAFlowNode().getFunction().refersTo(callee)
)
or
exists(Attribute a | call.getFunc() = a |
unique_root_method(result, a.getName())
or
exists(Name n | a.getObject() = n and n.getId() = "self" |
result.getScope() = this.getScope() and
result.getName() = a.getName()
)
)
)
}
/**
* Afferent Coupling
* the number of callables that depend on this method.
* This is sometimes called the "fan-in" of a method.
*/
int getAfferentCoupling() { result = count(FunctionMetrics m | m.getADependency() = this) }
/**
* Efferent Coupling
* the number of methods that this method depends on
* This is sometimes called the "fan-out" of a method.
*/
int getEfferentCoupling() { result = count(FunctionMetrics m | this.getADependency() = m) }
int getNumberOfParametersWithoutDefault() {
result =
this.getPositionalParameterCount() -
count(this.getDefinition().(FunctionExpr).getArgs().getADefault())
}
int getStatementNestingDepth() { result = max(Stmt s | s.getScope() = this | getNestingDepth(s)) }
int getNumberOfCalls() { result = count(Call c | c.getScope() = this) }
}
/** The metrics for a class */
class ClassMetrics extends Class {
/**
* Gets the total number of lines (including blank lines)
* from the definition to the end of the class
*/
int getNumberOfLines() { py_alllines(this, result) }
/** Gets the number of lines of code in the class */
int getNumberOfLinesOfCode() { py_codelines(this, result) }
/** Gets the number of lines of comments in the class */
int getNumberOfLinesOfComments() { py_commentlines(this, result) }
/** Gets the number of lines of docstrings in the class */
int getNumberOfLinesOfDocStrings() { py_docstringlines(this, result) }
private predicate dependsOn(Class other) {
other != this and
(
exists(FunctionMetrics f1, FunctionMetrics f2 | f1.getADependency() = f2 |
f1.getScope() = this and f2.getScope() = other
)
or
exists(Function f, Call c, ClassObject cls | c.getScope() = f and f.getScope() = this |
c.getFunc().refersTo(cls) and
cls.getPyClass() = other
)
)
}
/**
* The afferent coupling of a class is the number of classes that
* directly depend on it.
*/
int getAfferentCoupling() { result = count(ClassMetrics t | t.dependsOn(this)) }
/**
* The efferent coupling of a class is the number of classes that
* it directly depends on.
*/
int getEfferentCoupling() { result = count(ClassMetrics t | this.dependsOn(t)) }
int getInheritanceDepth() {
exists(ClassObject cls | cls.getPyClass() = this | result = max(classInheritanceDepth(cls)))
}
/* -------- CHIDAMBER AND KEMERER LACK OF COHESION IN METHODS ------------ */
/*
* The aim of this metric is to try and determine whether a class
* represents one abstraction (good) or multiple abstractions (bad).
* If a class represents multiple abstractions, it should be split
* up into multiple classes.
*
* In the Chidamber and Kemerer method, this is measured as follows:
* n1 = number of pairs of distinct methods in a class that do *not*
* have at least one commonly accessed field
* n2 = number of pairs of distinct methods in a class that do
* have at least one commonly accessed field
* lcom = ((n1 - n2)/2 max 0)
*
* We divide by 2 because each pair (m1,m2) is counted twice in n1 and n2.
*/
/** should function f be excluded from the cohesion computation? */
predicate ignoreLackOfCohesion(Function f) { f.isInitMethod() or f.isSpecialMethod() }
private predicate methodPair(Function m1, Function m2) {
m1.getScope() = this and
m2.getScope() = this and
not this.ignoreLackOfCohesion(m1) and
not this.ignoreLackOfCohesion(m2) and
m1 != m2
}
private predicate one_accesses_other(Function m1, Function m2) {
this.methodPair(m1, m2) and
(
exists(SelfAttributeRead sa |
sa.getName() = m1.getName() and
sa.getScope() = m2
)
or
exists(SelfAttributeRead sa |
sa.getName() = m2.getName() and
sa.getScope() = m1
)
)
}
/** do m1 and m2 access a common field or one calls the other? */
private predicate shareField(Function m1, Function m2) {
this.methodPair(m1, m2) and
exists(string name |
exists(SelfAttributeRead sa |
sa.getName() = name and
sa.getScope() = m1
) and
exists(SelfAttributeRead sa |
sa.getName() = name and
sa.getScope() = m2
)
)
}
private int similarMethodPairs() {
result =
count(Function m1, Function m2 |
this.methodPair(m1, m2) and
(this.shareField(m1, m2) or this.one_accesses_other(m1, m2))
) / 2
}
private int methodPairs() {
result = count(Function m1, Function m2 | this.methodPair(m1, m2)) / 2
}
/** return Chidamber and Kemerer Lack of Cohesion */
int getLackOfCohesionCK() {
exists(int n |
n = this.methodPairs() - 2 * this.similarMethodPairs() and
result = n.maximum(0)
)
}
private predicate similarMethodPairDag(Function m1, Function m2, int line) {
(this.shareField(m1, m2) or this.one_accesses_other(m1, m2)) and
line = m1.getLocation().getStartLine() and
line < m2.getLocation().getStartLine()
}
private predicate subgraph(Function m, int line) {
this.similarMethodPairDag(m, _, line) and not this.similarMethodPairDag(_, m, _)
or
exists(Function other | this.subgraph(other, line) |
this.similarMethodPairDag(other, m, _) or
this.similarMethodPairDag(m, other, _)
)
}
predicate unionSubgraph(Function m, int line) { line = min(int l | this.subgraph(m, l)) }
/** return Hitz and Montazeri Lack of Cohesion */
int getLackOfCohesionHM() { result = count(int line | this.unionSubgraph(_, line)) }
}
private int classInheritanceDepth(ClassObject cls) {
/* Prevent run-away recursion in case of circular inheritance */
not cls.getASuperType() = cls and
(
exists(ClassObject sup | cls.getABaseType() = sup | result = classInheritanceDepth(sup) + 1)
or
not exists(cls.getABaseType()) and
(
major_version() = 2 and result = 0
or
major_version() > 2 and result = 1
)
)
}
class ModuleMetrics extends Module {
/** Gets the total number of lines (including blank lines) in the module */
int getNumberOfLines() { py_alllines(this, result) }
/** Gets the number of lines of code in the module */
int getNumberOfLinesOfCode() { py_codelines(this, result) }
/** Gets the number of lines of comments in the module */
int getNumberOfLinesOfComments() { py_commentlines(this, result) }
/** Gets the number of lines of docstrings in the module */
int getNumberOfLinesOfDocStrings() { py_docstringlines(this, result) }
/**
* The afferent coupling of a class is the number of classes that
* directly depend on it.
*/
int getAfferentCoupling() { result = count(ModuleMetrics t | t.dependsOn(this)) }
/**
* The efferent coupling of a class is the number of classes that
* it directly depends on.
*/
int getEfferentCoupling() { result = count(ModuleMetrics t | this.dependsOn(t)) }
private predicate dependsOn(Module other) {
other != this and
(
exists(FunctionMetrics f1, FunctionMetrics f2 | f1.getADependency() = f2 |
f1.getEnclosingModule() = this and f2.getEnclosingModule() = other
)
or
exists(Function f, Call c, ClassObject cls | c.getScope() = f and f.getScope() = this |
c.getFunc().refersTo(cls) and
cls.getPyClass().getEnclosingModule() = other
)
)
}
}
/** Helpers for coupling */
predicate unique_root_method(Function func, string name) {
name = func.getName() and
not exists(FunctionObject f, FunctionObject other |
f.getFunction() = func and
other.getName() = name
|
not other.overrides(f)
)
}
predicate non_coupling_method(Function f) {
f.isSpecialMethod() or
f.isInitMethod() or
f.getName() = "close" or
f.getName() = "write" or
f.getName() = "read" or
f.getName() = "get" or
f.getName() = "set"
}
private int getNestingDepth(Stmt s) {
not exists(Stmt outer | outer.getASubStatement() = s) and result = 1
or
exists(Stmt outer | outer.getASubStatement() = s |
if s.(If).isElif() or s instanceof ExceptStmt
then
/* If statement is an `elif` or `except` then it is not indented relative to its parent */
result = getNestingDepth(outer)
else result = getNestingDepth(outer) + 1
)
}

285
python/ql/src/semmle/python/Module.qll
View File

@@ -1,285 +0,0 @@
import python
private import semmle.python.objects.ObjectAPI
private import semmle.python.objects.Modules
/**
* A module. This is the top level element in an AST, corresponding to a source file.
* It is also a Scope; the scope of global variables.
*/
class Module extends Module_, Scope, AstNode {
override string toString() {
result = this.getKind() + " " + this.getName()
or
/* No name is defined, which means that this module is not on an import path. So it must be a script */
not exists(this.getName()) and
not this.isPackage() and
result = "Script " + this.getFile().getShortName()
or
/* Package missing name, so just use the path instead */
not exists(this.getName()) and
this.isPackage() and
result = "Package at " + this.getPath().getAbsolutePath()
}
/**
* This method will be deprecated in the next release. Please use `getEnclosingScope()` instead.
* The enclosing scope of this module (always none)
*/
override Scope getScope() { none() }
/** The enclosing scope of this module (always none) */
override Scope getEnclosingScope() { none() }
/** Gets the statements forming the body of this module */
override StmtList getBody() { result = Module_.super.getBody() }
/** Gets the nth statement of this module */
override Stmt getStmt(int n) { result = Module_.super.getStmt(n) }
/** Gets a top-level statement in this module */
override Stmt getAStmt() { result = Module_.super.getAStmt() }
/** Gets the name of this module */
override string getName() {
result = Module_.super.getName() and legalDottedName(result)
or
not exists(Module_.super.getName()) and
result = moduleNameFromFile(this.getPath())
}
/** Gets the short name of the module. For example the short name of module x.y.z is 'z' */
string getShortName() {
result = this.getName().suffix(this.getPackage().getName().length() + 1)
or
result = this.getName() and not exists(this.getPackage())
}
/** Gets this module */
override Module getEnclosingModule() { result = this }
/** Gets the __init__ module of this module if the module is a package and it has an __init__ module */
Module getInitModule() {
/* this.isPackage() and */ result.getName() = this.getName() + ".__init__"
}
/** Whether this module is a package initializer */
predicate isPackageInit() { this.getName().matches("%\\_\\_init\\_\\_") and not this.isPackage() }
/** Gets a name exported by this module, that is the names that will be added to a namespace by 'from this-module import *' */
string getAnExport() {
py_exports(this, result)
or
exists(ModuleObjectInternal mod | mod.getSource() = this.getEntryNode() |
mod.(ModuleValue).exports(result)
)
}
/** Gets the source file for this module */
File getFile() { py_module_path(this, result) }
/** Gets the source file or folder for this module or package */
Container getPath() { py_module_path(this, result) }
/** Whether this is a package */
predicate isPackage() { this.getPath() instanceof Folder }
/** Gets the package containing this module (or parent package if this is a package) */
Module getPackage() {
this.getName().matches("%.%") and
result.getName() = getName().regexpReplaceAll("\\.[^.]*$", "")
}
/** Gets the name of the package containing this module */
string getPackageName() {
this.getName().matches("%.%") and
result = getName().regexpReplaceAll("\\.[^.]*$", "")
}
/** Gets the metrics for this module */
ModuleMetrics getMetrics() { result = this }
/**
* Use ModuleObject.getAnImportedModule() instead.
* Gets a module imported by this module
*/
deprecated Module getAnImportedModule() { result.getName() = this.getAnImportedModuleName() }
string getAnImportedModuleName() {
exists(Import i | i.getEnclosingModule() = this | result = i.getAnImportedModuleName())
or
exists(ImportStar i | i.getEnclosingModule() = this | result = i.getImportedModuleName())
}
override Location getLocation() {
py_scope_location(result, this)
or
not py_scope_location(_, this) and
locations_ast(result, this, 0, 0, 0, 0)
}
/** Gets a child module or package of this package */
Module getSubModule(string name) {
result.getPackage() = this and
name = result.getName().regexpReplaceAll(".*\\.", "")
}
/** Whether name is declared in the __all__ list of this module */
predicate declaredInAll(string name) {
exists(AssignStmt a, GlobalVariable all |
a.defines(all) and
a.getScope() = this and
all.getId() = "__all__" and
(
a.getValue().(List).getAnElt().(StrConst).getText() = name
or
a.getValue().(Tuple).getAnElt().(StrConst).getText() = name
)
)
}
override AstNode getAChildNode() { result = this.getAStmt() }
predicate hasFromFuture(string attr) {
exists(Import i, ImportMember im, ImportExpr ie, Alias a, Name name |
im.getModule() = ie and
ie.getName() = "__future__" and
a.getAsname() = name and
name.getId() = attr and
i.getASubExpression() = im and
i.getAName() = a and
i.getEnclosingModule() = this
)
}
/** Gets the path element from which this module was loaded. */
Container getLoadPath() { result = this.getPath().getImportRoot() }
/** Holds if this module is in the standard library for version `major.minor` */
predicate inStdLib(int major, int minor) { this.getLoadPath().isStdLibRoot(major, minor) }
/** Holds if this module is in the standard library */
predicate inStdLib() { this.getLoadPath().isStdLibRoot() }
override predicate containsInScope(AstNode inner) { Scope.super.containsInScope(inner) }
override predicate contains(AstNode inner) { Scope.super.contains(inner) }
/** Gets the kind of this module. */
override string getKind() {
if this.isPackage()
then result = "Package"
else (
not exists(Module_.super.getKind()) and result = "Module"
or
result = Module_.super.getKind()
)
}
}
bindingset[name]
private predicate legalDottedName(string name) {
name.regexpMatch("(\\p{L}|_)(\\p{L}|\\d|_)*(\\.(\\p{L}|_)(\\p{L}|\\d|_)*)*")
}
bindingset[name]
private predicate legalShortName(string name) { name.regexpMatch("(\\p{L}|_)(\\p{L}|\\d|_)*") }
/**
* Holds if `f` is potentially a source package.
* Does it have an __init__.py file (or --respect-init=False for Python 2) and is it within the source archive?
*/
private predicate isPotentialSourcePackage(Folder f) {
f.getRelativePath() != "" and
isPotentialPackage(f)
}
private predicate isPotentialPackage(Folder f) {
exists(f.getFile("__init__.py"))
or
py_flags_versioned("options.respect_init", "False", _) and major_version() = 2
}
private string moduleNameFromBase(Container file) {
isPotentialPackage(file) and result = file.getBaseName()
or
file instanceof File and result = file.getStem()
}
/**
* Holds if `file` may be transitively imported from a file that may serve as the entry point of
* the execution.
*/
private predicate transitively_imported_from_entry_point(File file) {
file.getExtension().matches("%py%") and
exists(File importer |
// Only consider files that are in the source archive
exists(importer.getRelativePath()) and
importer.getParent() = file.getParent() and
exists(ImportExpr i |
i.getLocation().getFile() = importer and
i.getName() = file.getStem() and
// Disregard relative imports
i.getLevel() = 0
)
|
importer.isPossibleEntryPoint() or transitively_imported_from_entry_point(importer)
)
}
string moduleNameFromFile(Container file) {
exists(string basename |
basename = moduleNameFromBase(file) and
legalShortName(basename)
|
result = moduleNameFromFile(file.getParent()) + "." + basename
or
// If `file` is a transitive import of a file that's executed directly, we allow references
// to it by its `basename`.
transitively_imported_from_entry_point(file) and
result = basename
)
or
isPotentialSourcePackage(file) and
result = file.getStem() and
(
not isPotentialSourcePackage(file.getParent()) or
not legalShortName(file.getParent().getBaseName())
)
or
result = file.getStem() and file.getParent() = file.getImportRoot()
or
result = file.getStem() and isStubRoot(file.getParent())
}
private predicate isStubRoot(Folder f) {
not f.getParent*().isImportRoot() and
f.getAbsolutePath().matches("%/data/python/stubs")
}
/**
* Holds if the Container `c` should be the preferred file or folder for
* the given name when performing imports.
* Trivially true for any container if it is the only one with its name.
* However, if there are several modules with the same name, then
* this is the module most likely to be imported under that name.
*/
predicate isPreferredModuleForName(Container c, string name) {
exists(int p |
p = min(int x | x = priorityForName(_, name)) and
p = priorityForName(c, name)
)
}
private int priorityForName(Container c, string name) {
name = moduleNameFromFile(c) and
(
// In the source
exists(c.getRelativePath()) and result = -1
or
// On an import path
exists(c.getImportRoot(result))
or
// Otherwise
result = 10000
)
}

239
python/ql/src/semmle/python/Operations.qll
View File

@@ -1,239 +0,0 @@
import python
/** Base class for operators */
class Operator extends Operator_ {
/** Gets the name of the special method used to implement this operator */
string getSpecialMethodName() { none() }
}
/* Unary Expression and its operators */
/** A unary expression: (`+x`), (`-x`) or (`~x`) */
class UnaryExpr extends UnaryExpr_ {
override Expr getASubExpression() { result = this.getOperand() }
}
/** A unary operator: `+`, `-`, `~` or `not` */
class Unaryop extends Unaryop_ {
/** Gets the name of the special method used to implement this operator */
string getSpecialMethodName() { none() }
}
/** An invert (`~`) unary operator */
class Invert extends Invert_ {
override string getSpecialMethodName() { result = "__invert__" }
}
/** A positive (`+`) unary operator */
class UAdd extends UAdd_ {
override string getSpecialMethodName() { result = "__pos__" }
}
/** A negation (`-`) unary operator */
class USub extends USub_ {
override string getSpecialMethodName() { result = "__neg__" }
}
/** A `not` unary operator */
class Not extends Not_ {
override string getSpecialMethodName() { none() }
}
/* Binary Operation and its operators */
/** A binary expression, such as `x + y` */
class BinaryExpr extends BinaryExpr_ {
override Expr getASubExpression() { result = this.getLeft() or result = this.getRight() }
}
/** A power (`**`) binary operator */
class Pow extends Pow_ {
override string getSpecialMethodName() { result = "__pow__" }
}
/** A right shift (`>>`) binary operator */
class RShift extends RShift_ {
override string getSpecialMethodName() { result = "__rshift__" }
}
/** A subtract (`-`) binary operator */
class Sub extends Sub_ {
override string getSpecialMethodName() { result = "__sub__" }
}
/** A bitwise and (`&`) binary operator */
class BitAnd extends BitAnd_ {
override string getSpecialMethodName() { result = "__and__" }
}
/** A bitwise or (`|`) binary operator */
class BitOr extends BitOr_ {
override string getSpecialMethodName() { result = "__or__" }
}
/** A bitwise exclusive-or (`^`) binary operator */
class BitXor extends BitXor_ {
override string getSpecialMethodName() { result = "__xor__" }
}
/** An add (`+`) binary operator */
class Add extends Add_ {
override string getSpecialMethodName() { result = "__add__" }
}
/** An (true) divide (`/`) binary operator */
class Div extends Div_ {
override string getSpecialMethodName() {
result = "__truediv__"
or
major_version() = 2 and result = "__div__"
}
}
/** An floor divide (`//`) binary operator */
class FloorDiv extends FloorDiv_ {
override string getSpecialMethodName() { result = "__floordiv__" }
}
/** A left shift (`<<`) binary operator */
class LShift extends LShift_ {
override string getSpecialMethodName() { result = "__lshift__" }
}
/** A modulo (`%`) binary operator, which includes string formatting */
class Mod extends Mod_ {
override string getSpecialMethodName() { result = "__mod__" }
}
/** A multiplication (`*`) binary operator */
class Mult extends Mult_ {
override string getSpecialMethodName() { result = "__mul__" }
}
/** A matrix multiplication (`@`) binary operator */
class MatMult extends MatMult_ {
override string getSpecialMethodName() { result = "__matmul__" }
}
/* Comparison Operation and its operators */
/** A comparison operation, such as `x<y` */
class Compare extends Compare_ {
override Expr getASubExpression() { result = this.getLeft() or result = this.getAComparator() }
/**
* Whether as part of this comparison 'left' is compared with 'right' using the operator 'op'.
* For example, the comparison `a<b<c` compares(`a`, `b`, `<`) and compares(`b`, `c`, `<`).
*/
predicate compares(Expr left, Cmpop op, Expr right) {
this.getLeft() = left and this.getComparator(0) = right and op = this.getOp(0)
or
exists(int n |
this.getComparator(n) = left and this.getComparator(n + 1) = right and op = this.getOp(n + 1)
)
}
}
/** List of comparison operators in a comparison */
class CmpopList extends CmpopList_ { }
/** A comparison operator */
abstract class Cmpop extends Cmpop_ {
string getSymbol() { none() }
string getSpecialMethodName() { none() }
}
/** A greater than (`>`) comparison operator */
class Gt extends Gt_ {
override string getSymbol() { result = ">" }
override string getSpecialMethodName() { result = "__gt__" }
}
/** A greater than or equals (`>=`) comparison operator */
class GtE extends GtE_ {
override string getSymbol() { result = ">=" }
override string getSpecialMethodName() { result = "__ge__" }
}
/** An `in` comparison operator */
class In extends In_ {
override string getSymbol() { result = "in" }
}
/** An `is` comparison operator */
class Is extends Is_ {
override string getSymbol() { result = "is" }
}
/** An `is not` comparison operator */
class IsNot extends IsNot_ {
override string getSymbol() { result = "is not" }
}
/** An equals (`==`) comparison operator */
class Eq extends Eq_ {
override string getSymbol() { result = "==" }
override string getSpecialMethodName() { result = "__eq__" }
}
/** A less than (`<`) comparison operator */
class Lt extends Lt_ {
override string getSymbol() { result = "<" }
override string getSpecialMethodName() { result = "__lt__" }
}
/** A less than or equals (`<=`) comparison operator */
class LtE extends LtE_ {
override string getSymbol() { result = "<=" }
override string getSpecialMethodName() { result = "__le__" }
}
/** A not equals (`!=`) comparison operator */
class NotEq extends NotEq_ {
override string getSymbol() { result = "!=" }
override string getSpecialMethodName() { result = "__ne__" }
}
/** An `not in` comparison operator */
class NotIn extends NotIn_ {
override string getSymbol() { result = "not in" }
}
/* Boolean Operation (and/or) and its operators */
/** A boolean shortcut (and/or) operation */
class BoolExpr extends BoolExpr_ {
override Expr getASubExpression() { result = this.getAValue() }
string getOperator() {
this.getOp() instanceof And and result = "and"
or
this.getOp() instanceof Or and result = "or"
}
/** Whether part evaluates to partIsTrue if this evaluates to wholeIsTrue */
predicate impliesValue(Expr part, boolean partIsTrue, boolean wholeIsTrue) {
if this.getOp() instanceof And
then (
wholeIsTrue = true and partIsTrue = true and part = this.getAValue()
or
wholeIsTrue = true and this.getAValue().(BoolExpr).impliesValue(part, partIsTrue, true)
) else (
wholeIsTrue = false and partIsTrue = false and part = this.getAValue()
or
wholeIsTrue = false and this.getAValue().(BoolExpr).impliesValue(part, partIsTrue, false)
)
}
}
/** A short circuit boolean operator, and/or */
class Boolop extends Boolop_ { }
/** An `and` boolean operator */
class And extends And_ { }
/** An `or` boolean operator */
class Or extends Or_ { }

687
python/ql/src/semmle/python/PrintAst.qll
View File

@@ -1,687 +0,0 @@
/**
* Provides queries to pretty-print a Python AST as a graph.
*
* By default, this will print the AST for all elements in the database. To change this behavior,
* extend `PrintAstConfiguration` and override `shouldPrint` to hold for only the elements
* you wish to view the AST for.
*/
import python
import semmle.python.RegexTreeView
private newtype TPrintAstConfiguration = MkPrintAstConfiguration()
/**
* The query can extend this class to control which elements are printed.
*/
class PrintAstConfiguration extends TPrintAstConfiguration {
/**
* Gets a textual representation of this `PrintAstConfiguration`.
*/
string toString() { result = "PrintAstConfiguration" }
/**
* Controls whether the `AstNode` should be considered for AST printing.
* By default it checks whether the `AstNode` `e` belongs to `Location` `l`.
*/
predicate shouldPrint(AstNode e, Location l) { l = e.getLocation() }
}
private predicate shouldPrint(AstNode e, Location l) {
exists(PrintAstConfiguration config | config.shouldPrint(e, l))
}
/** Holds if the given element does not need to be rendered in the AST. */
private predicate isNotNeeded(AstNode el) {
el.isArtificial()
or
el instanceof Module
or
exists(AstNode parent | isNotNeeded(parent) and not parent instanceof Module |
el = parent.getAChildNode()
)
}
/**
* Printed nodes.
*/
private newtype TPrintAstNode =
TElementNode(AstNode el) { shouldPrint(el, _) and not isNotNeeded(el) } or
TFunctionParamsNode(Function f) { shouldPrint(f, _) and not isNotNeeded(f) } or
TCallArgumentsNode(Call c) { shouldPrint(c, _) and not isNotNeeded(c) } or
TStmtListNode(StmtList list) {
shouldPrint(list.getAnItem(), _) and
not list = any(Module mod).getBody() and
not forall(AstNode child | child = list.getAnItem() | isNotNeeded(child)) and
exists(list.getAnItem())
} or
TRegExpTermNode(RegExpTerm term) {
exists(StrConst str | term.getRootTerm() = getParsedRegExp(str) and shouldPrint(str, _))
}
/**
* A node in the output tree.
*/
class PrintAstNode extends TPrintAstNode {
/**
* Gets a textual representation of this node in the PrintAst output tree.
*/
string toString() { none() }
/**
* Gets the child node at index `childIndex`. Child indices must be unique,
* but need not be contiguous.
*/
PrintAstNode getChild(int childIndex) { none() }
/**
* Gets a child of this node.
*/
final PrintAstNode getAChild() { result = getChild(_) }
/**
* Gets the parent of this node, if any.
*/
final PrintAstNode getParent() { result.getAChild() = this }
/**
* Gets the location of this node in the source code.
*/
Location getLocation() { none() }
/**
* Gets the value of the property of this node, where the name of the property
* is `key`.
*/
string getProperty(string key) {
key = "semmle.label" and
result = toString()
}
/**
* Gets the label for the edge from this node to the specified child. By
* default, this is just the index of the child, but subclasses can override
* this.
*/
string getChildEdgeLabel(int childIndex) {
exists(getChild(childIndex)) and
result = childIndex.toString()
}
}
/** A top-level AST node. */
class TopLevelPrintAstNode extends PrintAstNode {
TopLevelPrintAstNode() { not exists(this.getParent()) }
private int getOrder() {
this =
rank[result](TopLevelPrintAstNode n, Location l |
l = n.getLocation()
|
n
order by
l.getFile().getRelativePath(), l.getStartLine(), l.getStartColumn(), l.getEndLine(),
l.getEndColumn()
)
}
override string getProperty(string key) {
result = super.getProperty(key)
or
key = "semmle.order" and
result = this.getOrder().toString()
}
}
/**
* An `AstNode` printed in the print-viewer.
*
* This class can be overridden to define more specific behavior for some `AstNode`s.
* The `getChildNode` and `getStmtList` methods can be overridden to easily set up a child-parent relation between different `AstElementNode`s.
* Be very careful about overriding `getChild`, as `getChildNode` and `getStmtList` depend on the default behavior of `getChild`.
*/
class AstElementNode extends PrintAstNode, TElementNode {
AstNode element;
AstElementNode() { this = TElementNode(element) }
override string toString() {
result = "[" + PrettyPrinting::getQlClass(element) + "] " + PrettyPrinting::prettyPrint(element)
}
override Location getLocation() { result = element.getLocation() }
/**
* Gets the `AstNode` that is printed by this print node.
*/
final AstNode getAstNode() { result = element }
override PrintAstNode getChild(int childIndex) {
exists(AstNode el | result.(AstElementNode).getAstNode() = el |
el = this.getChildNode(childIndex) and not el = getStmtList(_, _).getAnItem()
)
or
// displaying all `StmtList` after the other children.
exists(int offset | offset = 1 + max([0, any(int index | exists(this.getChildNode(index)))]) |
exists(int index | childIndex = index + offset |
result.(StmtListNode).getList() = getStmtList(index, _)
)
)
}
/**
* Gets a child node for the AstNode that this print node represents.
*
* The default behavior in `getChild` uses `getChildNode` to easily define a parent-child relation between different `AstElementNode`s.
*/
AstNode getChildNode(int childIndex) { result = getChild(element, childIndex) }
/**
* Gets the `index`th `StmtList` that is a child of the `AstNode` that this print node represents.
* `label` is used for pretty-printing a label in the parent-child relation in the ast-viewer.
*
* The `StmtListNode` class and the `getChild` predicate uses `getStmtList` to define a parent-child relation with labels.
*
* `index` must be 0 or positive.
*/
StmtList getStmtList(int index, string label) { none() }
}
/**
* A print node for `Try` statements.
*/
class TryNode extends AstElementNode {
override Try element;
override StmtList getStmtList(int index, string label) {
index = 0 and result = element.getBody() and label = "body"
or
index = 1 and result = element.getOrelse() and label = "orelse"
or
index = 2 and result = element.getHandlers() and label = "handlers"
or
index = 3 and result = element.getFinalbody() and label = "final body"
}
}
/**
* A print node for `If` statements.
*/
class IfNode extends AstElementNode {
override If element;
override AstNode getChildNode(int childIndex) { childIndex = 0 and result = element.getTest() }
override StmtList getStmtList(int index, string label) {
index = 1 and result = element.getBody() and label = "body"
or
index = 2 and result = element.getOrelse() and label = "orelse"
}
}
/**
* A print node for classes.
*/
class ClassNode extends AstElementNode {
override Class element;
override StmtList getStmtList(int index, string label) {
index = 1 and result = element.getBody() and label = "body"
}
}
/**
* A print node for `ExceptStmt`.
*/
class ExceptNode extends AstElementNode {
override ExceptStmt element;
override StmtList getStmtList(int index, string label) {
index = 1 and result = element.getBody() and label = "body"
}
}
/**
* A print node for `With` statements.
*/
class WithNode extends AstElementNode {
override With element;
override StmtList getStmtList(int index, string label) {
index = 1 and result = element.getBody() and label = "body"
}
}
/**
* A print node for `For` statements.
*/
class ForPrintNode extends AstElementNode {
override For element;
override StmtList getStmtList(int index, string label) {
index = 1 and result = element.getBody() and label = "body"
or
index = 2 and result = element.getOrelse() and label = "orelse"
}
}
/**
* A print node for `While` statements.
*/
class WhilePrintNode extends AstElementNode {
override While element;
override StmtList getStmtList(int index, string label) {
index = 1 and result = element.getBody() and label = "body"
or
index = 2 and result = element.getOrelse() and label = "orelse"
}
}
/**
* A print node for `StmtList`.
* A `StmtListNode` is always a child of an `AstElementNode`,
* and the child-parent relation is defined by the `getStmtList` predicate in `AstElementNode`.
*
* The label for a `StmtList` is decided based on the result from the `getStmtList` predicate in `AstElementNode`.
*/
class StmtListNode extends PrintAstNode, TStmtListNode {
StmtList list;
StmtListNode() {
this = TStmtListNode(list) and
list = any(AstElementNode node).getStmtList(_, _)
}
/**
* Gets the `StmtList` that this print node represents.
*/
StmtList getList() { result = list }
private string getLabel() { this.getList() = any(AstElementNode node).getStmtList(_, result) }
override string toString() { result = "(StmtList) " + getLabel() }
override PrintAstNode getChild(int childIndex) {
exists(AstNode el | result.(AstElementNode).getAstNode() = el | el = list.getItem(childIndex))
}
}
/**
* A print node for a `Call`.
*
* The arguments to this call are aggregated into a `CallArgumentsNode`.
*/
class CallPrintNode extends AstElementNode {
override Call element;
override PrintAstNode getChild(int childIndex) {
childIndex = 0 and result.(AstElementNode).getAstNode() = element.getFunc()
or
childIndex = 1 and result.(CallArgumentsNode).getCall() = element
}
}
/**
* A synthetic print node for the arguments to `call`.
*/
class CallArgumentsNode extends PrintAstNode, TCallArgumentsNode {
Call call;
CallArgumentsNode() { this = TCallArgumentsNode(call) }
/**
* Gets the call for which this print node represents the arguments.
*/
Call getCall() { result = call }
override string toString() { result = "(arguments)" }
override PrintAstNode getChild(int childIndex) {
result.(AstElementNode).getAstNode() = getChild(call, childIndex) and
not result.(AstElementNode).getAstNode() = call.getFunc()
}
}
/**
* A print node for a `Function`.
*/
class FunctionNode extends AstElementNode {
override Function element;
override PrintAstNode getChild(int childIndex) {
exists(FunctionParamsNode paramsNode | paramsNode.getFunction() = element |
childIndex = 0 and result = paramsNode
or
result = AstElementNode.super.getChild(childIndex) and
// parameters is handled above
not result.(AstElementNode).getAstNode() =
paramsNode.getChild(_).(AstElementNode).getAstNode() and
// The default of a Parameter is handled by `ParameterNode`
not result.(AstElementNode).getAstNode() = any(Parameter param).getDefault() and
// The annotation is a parameter is handled by `ParameterNode`.
not result.(AstElementNode).getAstNode() = any(Parameter param).getAnnotation()
)
}
override StmtList getStmtList(int index, string label) {
index = 1 and result = element.getBody() and label = "body"
}
}
/**
* A print node for a `FunctionDef`.
*/
class FunctionDefNode extends AstElementNode {
override FunctionDef element;
override AstNode getChildNode(int childIndex) {
childIndex = 0 and result = element.getTarget(0)
or
childIndex = 1 and result = element.getValue()
}
}
/**
* A print node for the parameters in `func`.
*/
class FunctionParamsNode extends PrintAstNode, TFunctionParamsNode {
Function func;
FunctionParamsNode() { this = TFunctionParamsNode(func) }
/**
* Gets the `Function` that this print node represents.
*/
Function getFunction() { result = func }
override string toString() { result = "(parameters)" }
override PrintAstNode getChild(int childIndex) {
// everything that is not a stmt is a parameter.
exists(AstNode el | result.(AstElementNode).getAstNode() = el |
el = getChild(func, childIndex) and not el = func.getAStmt()
)
}
}
/**
* A print node for a `Parameter`.
*
* This print node has the annotation and default value of the `Parameter` as children.
* The type annotation and default value would by default exist as children of the parent `Function`.
*/
class ParameterNode extends AstElementNode {
Parameter param;
ParameterNode() { this.getAstNode() = param.asName() or this.getAstNode() = param.asTuple() }
override AstNode getChildNode(int childIndex) {
childIndex = 0 and result = param.getAnnotation()
or
childIndex = 1 and result = param.getDefault()
}
}
/**
* A print node for a `StrConst`.
*
* The string has a child, if the child is used as a regular expression,
* which is the root of the regular expression.
*/
class StrConstNode extends AstElementNode {
override StrConst element;
override PrintAstNode getChild(int childIndex) {
childIndex = 0 and result.(RegExpTermNode).getTerm() = getParsedRegExp(element)
}
}
/**
* A print node for a regular expression term.
*/
class RegExpTermNode extends TRegExpTermNode, PrintAstNode {
RegExpTerm term;
RegExpTermNode() { this = TRegExpTermNode(term) }
/** Gets the `RegExpTerm` for this node. */
RegExpTerm getTerm() { result = term }
override PrintAstNode getChild(int childIndex) {
result.(RegExpTermNode).getTerm() = term.getChild(childIndex)
}
override string toString() {
result = "[" + strictconcat(term.getPrimaryQLClass(), " | ") + "] " + term.toString()
}
override Location getLocation() { result = term.getLocation() }
}
/**
* Gets the `i`th child from `node` ordered by location.
*/
private AstNode getChild(AstNode node, int i) {
shouldPrint(node, _) and
result =
rank[i](AstNode child |
child = node.getAChildNode()
|
child
order by
child.getLocation().getStartLine(), child.getLocation().getStartColumn(),
child.getLocation().getEndLine(), child.getLocation().getEndColumn()
)
}
/**
* A module for pretty-printing some `AstNode`s.
*/
private module PrettyPrinting {
/**
* Gets the QL class for the `AstNode` `a`.
* Most `AstNode`s print their QL class in the `toString()` method, however there are exceptions.
* These exceptions are handled in the `getQlCustomClass` predicate.
*/
string getQlClass(AstNode a) {
shouldPrint(a, _) and
(
not exists(getQlCustomClass(a)) and result = strictconcat(a.toString(), " | ")
or
result = strictconcat(getQlCustomClass(a), " | ")
)
}
/**
* Gets the QL class for `AstNode`s where the `toString` method does not print the QL class.
*/
string getQlCustomClass(AstNode a) {
shouldPrint(a, _) and
(
a instanceof Name and
result = "Name" and
not a instanceof Parameter and
not a instanceof NameConstant
or
a instanceof Parameter and result = "Parameter"
or
a instanceof PlaceHolder and result = "PlaceHolder"
or
a instanceof Function and result = "Function"
or
a instanceof Class and result = "Class"
or
a instanceof Call and result = "Call"
or
a instanceof NameConstant and result = "NameConstant"
)
}
/**
* Gets a human-readable representation of the `AstNode` `a`, or the empty string.
*
* Has exactly one result for every `AstNode`.
*/
string prettyPrint(AstNode a) {
shouldPrint(a, _) and
(
// this strictconcat should not be needed.
// However, the printAst feature breaks if this predicate has more than one result for an `AstNode`, so the strictconcat stays.
result = strictconcat(reprRec(a), " | ")
or
not exists(reprRec(a)) and
result = ""
)
}
/**
* Gets a human-readable representation of the given `AstNode`.
*
* Only has a result for some `AstNode`s.
*
* The monotonicity of this recursive predicate is kept by defining the non-recursive cases inside the `reprBase` predicate,
* and then using `reprBase` when there is a negative edge.
*/
private string reprRec(AstNode a) {
shouldPrint(a, _) and
not isNotNeeded(a) and
(
// For NameNodes, we just use the underlying variable name
result = reprBase(a)
or
exists(Expr obj |
obj = a.(Attribute).getObject() // Attribute .getname .getObject
|
// Attributes of the form `name.name2`
result = reprBase(obj) + "." + a.(Attribute).getName()
or
// Attributes where the object is a more complicated expression
not exists(reprBase(obj)) and
result = "(...)." + a.(Attribute).getName()
)
or
result = "import " + reprRec(a.(Import).getName(_).getAsname())
or
exists(Keyword keyword | keyword = a |
result = keyword.getArg() + "=" + reprRec(keyword.getValue())
)
or
result = reprRec(a.(Call).getFunc()) + "(" + printArgs(a) + ")"
or
not exists(printArgs(a)) and result = reprRec(a.(Call).getFunc()) + "(...)"
or
result = "try " + reprRec(a.(Try).getBody().getItem(0))
or
result = "if " + reprRec(a.(If).getTest()) + ":"
or
result = reprRec(a.(Compare).getLeft()) + " " + a.(Compare).getOp(0).getSymbol() + " ..."
or
result = a.(Subscript).getObject() + "[" + reprRec(a.(Subscript).getIndex()) + "]"
or
exists(Assign asn | asn = a |
strictcount(asn.getTargets()) = 1 and
result = reprRec(a.(Assign).getTarget(0)) + " = " + reprRec(asn.getValue())
)
or
result = "return " + reprRec(a.(Return).getValue())
or
result = reprRec(a.(ExprStmt).getValue())
or
exists(BoolExpr b, string op |
a = b and
(
b.getOp() instanceof And and op = "and"
or
b.getOp() instanceof Or and op = "or"
)
|
result = reprRec(b.getValue(0)) + " " + op + " " + reprRec(b.getValue(1))
)
)
}
/**
* Gets a comma separated pretty printed list of the arguments in `call`.
*/
string printArgs(Call call) {
not exists(call.getAnArg()) and result = ""
or
result = strictconcat(int i | | reprBase(call.getArg(i)), ", ")
}
/**
* Gets a human-readable representation of the given `AstNode`.
* Is only defined for `AstNode`s for which a human-readable representation can be created without using recursion.
*/
private string reprBase(AstNode a) {
shouldPrint(a, _) and
not isNotNeeded(a) and
(
result = a.(Name).getId()
or
result = a.(PlaceHolder).toString()
or
result = "class " + a.(ClassExpr).getName()
or
result = "class " + a.(Class).getName()
or
result = a.(StrConst).getText()
or
result = "yield " + a.(Yield).getValue()
or
result = "yield from " + a.(YieldFrom).getValue()
or
result = "*" + a.(Starred).getValue()
or
result = "`" + a.(Repr).getValue() + "`"
or
a instanceof Ellipsis and result = "..."
or
result = a.(Num).getText()
or
result = a.(NegativeIntegerLiteral).getValue().toString()
or
result = a.(NameConstant).toString()
or
result = "await " + a.(Await).getValue()
or
result = "function " + a.(FunctionExpr).getName() + "(...)"
or
result = "function " + a.(Function).getName() + "(...)"
or
a instanceof List and result = "[...]"
or
a instanceof Set and result = "{...}"
or
a instanceof Continue and result = "continue"
or
a instanceof Break and result = "break"
or
a instanceof Pass and result = "pass"
)
}
}
/** Holds if `node` belongs to the output tree, and its property `key` has the given `value`. */
query predicate nodes(PrintAstNode node, string key, string value) { value = node.getProperty(key) }
/**
* Holds if `target` is a child of `source` in the AST, and property `key` of the edge has the
* given `value`.
*/
query predicate edges(PrintAstNode source, PrintAstNode target, string key, string value) {
exists(int childIndex |
target = source.getChild(childIndex) and
(
key = "semmle.label" and value = source.getChildEdgeLabel(childIndex)
or
key = "semmle.order" and value = childIndex.toString()
)
)
}
/** Holds if property `key` of the graph has the given `value`. */
query predicate graphProperties(string key, string value) {
key = "semmle.graphKind" and value = "tree"
}

964
python/ql/src/semmle/python/RegexTreeView.qll
View File

@@ -1,964 +0,0 @@
/** Provides a class hierarchy corresponding to a parse tree of regular expressions. */
import python
private import semmle.python.regex
/**
* An element containing a regular expression term, that is, either
* a string literal (parsed as a regular expression)
* or another regular expression term.
*/
newtype TRegExpParent =
/** A string literal used as a regular expression */
TRegExpLiteral(Regex re) or
/** A quantified term */
TRegExpQuantifier(Regex re, int start, int end) { re.qualifiedItem(start, end, _, _) } or
/** A sequence term */
TRegExpSequence(Regex re, int start, int end) { re.sequence(start, end) } or
/** An alternatio term */
TRegExpAlt(Regex re, int start, int end) { re.alternation(start, end) } or
/** A character class term */
TRegExpCharacterClass(Regex re, int start, int end) { re.charSet(start, end) } or
/** A character range term */
TRegExpCharacterRange(Regex re, int start, int end) { re.charRange(_, start, _, _, end) } or
/** A group term */
TRegExpGroup(Regex re, int start, int end) { re.group(start, end) } or
/** A special character */
TRegExpSpecialChar(Regex re, int start, int end) { re.specialCharacter(start, end, _) } or
/** A normal character */
TRegExpNormalChar(Regex re, int start, int end) { re.normalCharacter(start, end) } or
/** A back reference */
TRegExpBackRef(Regex re, int start, int end) { re.backreference(start, end) }
/**
* An element containing a regular expression term, that is, either
* a string literal (parsed as a regular expression)
* or another regular expression term.
*/
class RegExpParent extends TRegExpParent {
string toString() { result = "RegExpParent" }
/** Gets the `i`th child term. */
abstract RegExpTerm getChild(int i);
/** Gets a child term . */
RegExpTerm getAChild() { result = getChild(_) }
/** Gets the number of child terms. */
int getNumChild() { result = count(getAChild()) }
/** Gets the associated regex. */
abstract Regex getRegex();
}
/** A string literal used as a regular expression */
class RegExpLiteral extends TRegExpLiteral, RegExpParent {
Regex re;
RegExpLiteral() { this = TRegExpLiteral(re) }
override RegExpTerm getChild(int i) { i = 0 and result.getRegex() = re and result.isRootTerm() }
predicate isDotAll() { re.getAMode() = "DOTALL" }
override Regex getRegex() { result = re }
string getPrimaryQLClass() { result = "RegExpLiteral" }
}
/**
* A regular expression term, that is, a syntactic part of a regular expression.
*/
class RegExpTerm extends RegExpParent {
Regex re;
int start;
int end;
RegExpTerm() {
this = TRegExpAlt(re, start, end)
or
this = TRegExpBackRef(re, start, end)
or
this = TRegExpCharacterClass(re, start, end)
or
this = TRegExpCharacterRange(re, start, end)
or
this = TRegExpNormalChar(re, start, end)
or
this = TRegExpGroup(re, start, end)
or
this = TRegExpQuantifier(re, start, end)
or
this = TRegExpSequence(re, start, end) and
exists(seqChild(re, start, end, 1)) // if a sequence does not have more than one element, it should be treated as that element instead.
or
this = TRegExpSpecialChar(re, start, end)
}
/**
* Gets the outermost term of this regular expression.
*/
RegExpTerm getRootTerm() {
this.isRootTerm() and result = this
or
result = getParent().(RegExpTerm).getRootTerm()
}
/**
* Holds if this term is part of a string literal
* that is interpreted as a regular expression.
*/
predicate isUsedAsRegExp() { any() }
/**
* Holds if this is the root term of a regular expression.
*/
predicate isRootTerm() { start = 0 and end = re.getText().length() }
override RegExpTerm getChild(int i) {
result = this.(RegExpAlt).getChild(i)
or
result = this.(RegExpBackRef).getChild(i)
or
result = this.(RegExpCharacterClass).getChild(i)
or
result = this.(RegExpCharacterRange).getChild(i)
or
result = this.(RegExpNormalChar).getChild(i)
or
result = this.(RegExpGroup).getChild(i)
or
result = this.(RegExpQuantifier).getChild(i)
or
result = this.(RegExpSequence).getChild(i)
or
result = this.(RegExpSpecialChar).getChild(i)
}
/**
* Gets the parent term of this regular expression term, or the
* regular expression literal if this is the root term.
*/
RegExpParent getParent() { result.getAChild() = this }
override Regex getRegex() { result = re }
/** Gets the offset at which this term starts. */
int getStart() { result = start }
/** Gets the offset at which this term ends. */
int getEnd() { result = end }
override string toString() { result = re.getText().substring(start, end) }
/**
* Gets the location of the surrounding regex, as locations inside the regex do not exist.
* To get location information corresponding to the term inside the regex,
* use `hasLocationInfo`.
*/
Location getLocation() { result = re.getLocation() }
/** Holds if this term is found at the specified location offsets. */
predicate hasLocationInfo(
string filepath, int startline, int startcolumn, int endline, int endcolumn
) {
exists(int re_start, int re_end |
re.getLocation().hasLocationInfo(filepath, startline, re_start, endline, re_end) and
startcolumn = re_start + start + 4 and
endcolumn = re_start + end + 3
)
}
/** Gets the file in which this term is found. */
File getFile() { result = this.getLocation().getFile() }
/** Gets the raw source text of this term. */
string getRawValue() { result = this.toString() }
/** Gets the string literal in which this term is found. */
RegExpLiteral getLiteral() { result = TRegExpLiteral(re) }
/** Gets the regular expression term that is matched (textually) before this one, if any. */
RegExpTerm getPredecessor() {
exists(RegExpTerm parent | parent = getParent() |
result = parent.(RegExpSequence).previousElement(this)
or
not exists(parent.(RegExpSequence).previousElement(this)) and
not parent instanceof RegExpSubPattern and
result = parent.getPredecessor()
)
}
/** Gets the regular expression term that is matched (textually) after this one, if any. */
RegExpTerm getSuccessor() {
exists(RegExpTerm parent | parent = getParent() |
result = parent.(RegExpSequence).nextElement(this)
or
not exists(parent.(RegExpSequence).nextElement(this)) and
not parent instanceof RegExpSubPattern and
result = parent.getSuccessor()
)
}
/** Gets the primary QL class for this term. */
string getPrimaryQLClass() { result = "RegExpTerm" }
}
/**
* A quantified regular expression term.
*
* Example:
*
* ```
* ((ECMA|Java)[sS]cript)*
* ```
*/
class RegExpQuantifier extends RegExpTerm, TRegExpQuantifier {
int part_end;
boolean maybe_empty;
boolean may_repeat_forever;
RegExpQuantifier() {
this = TRegExpQuantifier(re, start, end) and
re.qualifiedPart(start, part_end, end, maybe_empty, may_repeat_forever)
}
override RegExpTerm getChild(int i) {
i = 0 and
result.getRegex() = re and
result.getStart() = start and
result.getEnd() = part_end
}
predicate mayRepeatForever() { may_repeat_forever = true }
string getQualifier() { result = re.getText().substring(part_end, end) }
override string getPrimaryQLClass() { result = "RegExpQuantifier" }
}
/**
* A regular expression term that permits unlimited repetitions.
*/
class InfiniteRepetitionQuantifier extends RegExpQuantifier {
InfiniteRepetitionQuantifier() { this.mayRepeatForever() }
}
/**
* A star-quantified term.
*
* Example:
*
* ```
* \w*
* ```
*/
class RegExpStar extends InfiniteRepetitionQuantifier {
RegExpStar() { this.getQualifier().charAt(0) = "*" }
override string getPrimaryQLClass() { result = "RegExpStar" }
}
/**
* A plus-quantified term.
*
* Example:
*
* ```
* \w+
* ```
*/
class RegExpPlus extends InfiniteRepetitionQuantifier {
RegExpPlus() { this.getQualifier().charAt(0) = "+" }
override string getPrimaryQLClass() { result = "RegExpPlus" }
}
/**
* An optional term.
*
* Example:
*
* ```
* ;?
* ```
*/
class RegExpOpt extends RegExpQuantifier {
RegExpOpt() { this.getQualifier().charAt(0) = "?" }
override string getPrimaryQLClass() { result = "RegExpOpt" }
}
/**
* A range-quantified term
*
* Examples:
*
* ```
* \w{2,4}
* \w{2,}
* \w{2}
* ```
*/
class RegExpRange extends RegExpQuantifier {
string upper;
string lower;
RegExpRange() { re.multiples(part_end, end, lower, upper) }
string getUpper() { result = upper }
string getLower() { result = lower }
/**
* Gets the upper bound of the range, if any.
*
* If there is no upper bound, any number of repetitions is allowed.
* For a term of the form `r{lo}`, both the lower and the upper bound
* are `lo`.
*/
int getUpperBound() { result = this.getUpper().toInt() }
/** Gets the lower bound of the range. */
int getLowerBound() { result = this.getLower().toInt() }
override string getPrimaryQLClass() { result = "RegExpRange" }
}
/**
* A sequence term.
*
* Example:
*
* ```
* (ECMA|Java)Script
* ```
*
* This is a sequence with the elements `(ECMA|Java)` and `Script`.
*/
class RegExpSequence extends RegExpTerm, TRegExpSequence {
RegExpSequence() {
this = TRegExpSequence(re, start, end) and
exists(seqChild(re, start, end, 1)) // if a sequence does not have more than one element, it should be treated as that element instead.
}
override RegExpTerm getChild(int i) { result = seqChild(re, start, end, i) }
/** Gets the element preceding `element` in this sequence. */
RegExpTerm previousElement(RegExpTerm element) { element = nextElement(result) }
/** Gets the element following `element` in this sequence. */
RegExpTerm nextElement(RegExpTerm element) {
exists(int i |
element = this.getChild(i) and
result = this.getChild(i + 1)
)
}
override string getPrimaryQLClass() { result = "RegExpSequence" }
}
pragma[nomagic]
private int seqChildEnd(Regex re, int start, int end, int i) {
result = seqChild(re, start, end, i).getEnd()
}
// moved out so we can use it in the charpred
private RegExpTerm seqChild(Regex re, int start, int end, int i) {
re.sequence(start, end) and
(
i = 0 and
result.getRegex() = re and
result.getStart() = start and
exists(int itemEnd |
re.item(start, itemEnd) and
result.getEnd() = itemEnd
)
or
i > 0 and
result.getRegex() = re and
exists(int itemStart | itemStart = seqChildEnd(re, start, end, i - 1) |
result.getStart() = itemStart and
re.item(itemStart, result.getEnd())
)
)
}
/**
* An alternative term, that is, a term of the form `a|b`.
*
* Example:
*
* ```
* ECMA|Java
* ```
*/
class RegExpAlt extends RegExpTerm, TRegExpAlt {
RegExpAlt() { this = TRegExpAlt(re, start, end) }
override RegExpTerm getChild(int i) {
i = 0 and
result.getRegex() = re and
result.getStart() = start and
exists(int part_end |
re.alternationOption(start, end, start, part_end) and
result.getEnd() = part_end
)
or
i > 0 and
result.getRegex() = re and
exists(int part_start |
part_start = this.getChild(i - 1).getEnd() + 1 // allow for the |
|
result.getStart() = part_start and
re.alternationOption(start, end, part_start, result.getEnd())
)
}
override string getPrimaryQLClass() { result = "RegExpAlt" }
}
/**
* An escaped regular expression term, that is, a regular expression
* term starting with a backslash, which is not a backreference.
*
* Example:
*
* ```
* \.
* \w
* ```
*/
class RegExpEscape extends RegExpNormalChar {
RegExpEscape() { re.escapedCharacter(start, end) }
/**
* Gets the name of the escaped; for example, `w` for `\w`.
* TODO: Handle named escapes.
*/
override string getValue() {
this.isIdentityEscape() and result = this.getUnescaped()
or
this.getUnescaped() = "n" and result = "\n"
or
this.getUnescaped() = "r" and result = "\r"
or
this.getUnescaped() = "t" and result = "\t"
or
// TODO: Find a way to include a formfeed character
// this.getUnescaped() = "f" and result = " "
// or
isUnicode() and
result = getUnicode()
}
predicate isIdentityEscape() { not this.getUnescaped() in ["n", "r", "t", "f"] }
override string getPrimaryQLClass() { result = "RegExpEscape" }
string getUnescaped() { result = this.getText().suffix(1) }
/**
* Gets the text for this escape. That is e.g. "\w".
*/
private string getText() { result = re.getText().substring(start, end) }
/**
* Holds if this is a unicode escape.
*/
private predicate isUnicode() { getText().prefix(2) = ["\\u", "\\U"] }
/**
* Gets the unicode char for this escape.
* E.g. for `\u0061` this returns "a".
*/
private string getUnicode() {
exists(int codepoint | codepoint = sum(this.getHexValueFromUnicode(_)) |
result = codepoint.toUnicode()
)
}
/**
* Gets int value for the `index`th char in the hex number of the unicode escape.
* E.g. for `\u0061` and `index = 2` this returns 96 (the number `6` interpreted as hex).
*/
private int getHexValueFromUnicode(int index) {
this.isUnicode() and
exists(string hex, string char | hex = this.getText().suffix(2) |
char = hex.charAt(index) and
result = 16.pow(hex.length() - index - 1) * toHex(char)
)
}
}
/**
* Gets the hex number for the `hex` char.
*/
private int toHex(string hex) {
hex = [0 .. 9].toString() and
result = hex.toInt()
or
result = 10 and hex = ["a", "A"]
or
result = 11 and hex = ["b", "B"]
or
result = 12 and hex = ["c", "C"]
or
result = 13 and hex = ["d", "D"]
or
result = 14 and hex = ["e", "E"]
or
result = 15 and hex = ["f", "F"]
}
/**
* A character class escape in a regular expression.
* That is, an escaped charachter that denotes multiple characters.
*
* Examples:
*
* ```
* \w
* \S
* ```
*/
class RegExpCharacterClassEscape extends RegExpEscape {
// string value;
RegExpCharacterClassEscape() {
// value = re.getText().substring(start + 1, end) and
// value in ["d", "D", "s", "S", "w", "W"]
this.getValue() in ["d", "D", "s", "S", "w", "W"]
}
/** Gets the name of the character class; for example, `w` for `\w`. */
// override string getValue() { result = value }
override RegExpTerm getChild(int i) { none() }
override string getPrimaryQLClass() { result = "RegExpCharacterClassEscape" }
}
/**
* A character class in a regular expression.
*
* Examples:
*
* ```
* [a-z_]
* [^<>&]
* ```
*/
class RegExpCharacterClass extends RegExpTerm, TRegExpCharacterClass {
RegExpCharacterClass() { this = TRegExpCharacterClass(re, start, end) }
predicate isInverted() { re.getChar(start + 1) = "^" }
string getCharThing(int i) { result = re.getChar(i + start) }
predicate isUniversalClass() {
// [^]
isInverted() and not exists(getAChild())
or
// [\w\W] and similar
not isInverted() and
exists(string cce1, string cce2 |
cce1 = getAChild().(RegExpCharacterClassEscape).getValue() and
cce2 = getAChild().(RegExpCharacterClassEscape).getValue()
|
cce1 != cce2 and cce1.toLowerCase() = cce2.toLowerCase()
)
}
override RegExpTerm getChild(int i) {
i = 0 and
result.getRegex() = re and
exists(int itemStart, int itemEnd |
result.getStart() = itemStart and
re.char_set_start(start, itemStart) and
re.char_set_child(start, itemStart, itemEnd) and
result.getEnd() = itemEnd
)
or
i > 0 and
result.getRegex() = re and
exists(int itemStart | itemStart = this.getChild(i - 1).getEnd() |
result.getStart() = itemStart and
re.char_set_child(start, itemStart, result.getEnd())
)
}
override string getPrimaryQLClass() { result = "RegExpCharacterClass" }
}
/**
* A character range in a character class in a regular expression.
*
* Example:
*
* ```
* a-z
* ```
*/
class RegExpCharacterRange extends RegExpTerm, TRegExpCharacterRange {
int lower_end;
int upper_start;
RegExpCharacterRange() {
this = TRegExpCharacterRange(re, start, end) and
re.charRange(_, start, lower_end, upper_start, end)
}
predicate isRange(string lo, string hi) {
lo = re.getText().substring(start, lower_end) and
hi = re.getText().substring(upper_start, end)
}
override RegExpTerm getChild(int i) {
i = 0 and
result.getRegex() = re and
result.getStart() = start and
result.getEnd() = lower_end
or
i = 1 and
result.getRegex() = re and
result.getStart() = upper_start and
result.getEnd() = end
}
override string getPrimaryQLClass() { result = "RegExpCharacterRange" }
}
/**
* A normal character in a regular expression, that is, a character
* without special meaning. This includes escaped characters.
*
* Examples:
* ```
* t
* \t
* ```
*/
class RegExpNormalChar extends RegExpTerm, TRegExpNormalChar {
RegExpNormalChar() { this = TRegExpNormalChar(re, start, end) }
predicate isCharacter() { any() }
string getValue() { result = re.getText().substring(start, end) }
override RegExpTerm getChild(int i) { none() }
override string getPrimaryQLClass() { result = "RegExpNormalChar" }
}
/**
* A constant regular expression term, that is, a regular expression
* term matching a single string. Currently, this will always be a single character.
*
* Example:
*
* ```
* a
* ```
*/
class RegExpConstant extends RegExpTerm {
string value;
RegExpConstant() {
this = TRegExpNormalChar(re, start, end) and
not this instanceof RegExpCharacterClassEscape and
// exclude chars in qualifiers
// TODO: push this into regex library
not exists(int qstart, int qend | re.qualifiedPart(_, qstart, qend, _, _) |
qstart <= start and end <= qend
) and
value = this.(RegExpNormalChar).getValue()
// This will never hold
// or
// this = TRegExpSpecialChar(re, start, end) and
// re.inCharSet(start) and
// value = this.(RegExpSpecialChar).getChar()
}
predicate isCharacter() { any() }
string getValue() { result = value }
override RegExpTerm getChild(int i) { none() }
override string getPrimaryQLClass() { result = "RegExpConstant" }
}
/**
* A grouped regular expression.
*
* Examples:
*
* ```
* (ECMA|Java)
* (?:ECMA|Java)
* (?<quote>['"])
* ```
*/
class RegExpGroup extends RegExpTerm, TRegExpGroup {
RegExpGroup() { this = TRegExpGroup(re, start, end) }
/**
* Gets the index of this capture group within the enclosing regular
* expression literal.
*
* For example, in the regular expression `/((a?).)(?:b)/`, the
* group `((a?).)` has index 1, the group `(a?)` nested inside it
* has index 2, and the group `(?:b)` has no index, since it is
* not a capture group.
*/
int getNumber() { result = re.getGroupNumber(start, end) }
/** Holds if this is a named capture group. */
predicate isNamed() { exists(this.getName()) }
/** Gets the name of this capture group, if any. */
string getName() { result = re.getGroupName(start, end) }
predicate isCharacter() { any() }
string getValue() { result = re.getText().substring(start, end) }
override RegExpTerm getChild(int i) {
result.getRegex() = re and
i = 0 and
re.groupContents(start, end, result.getStart(), result.getEnd())
}
override string getPrimaryQLClass() { result = "RegExpGroup" }
}
/**
* A special character in a regular expression.
*
* Examples:
* ```
* ^
* $
* .
* ```
*/
class RegExpSpecialChar extends RegExpTerm, TRegExpSpecialChar {
string char;
RegExpSpecialChar() {
this = TRegExpSpecialChar(re, start, end) and
re.specialCharacter(start, end, char)
}
predicate isCharacter() { any() }
string getChar() { result = char }
override RegExpTerm getChild(int i) { none() }
override string getPrimaryQLClass() { result = "RegExpSpecialChar" }
}
/**
* A dot regular expression.
*
* Example:
*
* ```
* .
* ```
*/
class RegExpDot extends RegExpSpecialChar {
RegExpDot() { this.getChar() = "." }
override string getPrimaryQLClass() { result = "RegExpDot" }
}
/**
* A dollar assertion `$` matching the end of a line.
*
* Example:
*
* ```
* $
* ```
*/
class RegExpDollar extends RegExpSpecialChar {
RegExpDollar() { this.getChar() = "$" }
override string getPrimaryQLClass() { result = "RegExpDollar" }
}
/**
* A caret assertion `^` matching the beginning of a line.
*
* Example:
*
* ```
* ^
* ```
*/
class RegExpCaret extends RegExpSpecialChar {
RegExpCaret() { this.getChar() = "^" }
override string getPrimaryQLClass() { result = "RegExpCaret" }
}
/**
* A zero-width match, that is, either an empty group or an assertion.
*
* Examples:
* ```
* ()
* (?=\w)
* ```
*/
class RegExpZeroWidthMatch extends RegExpGroup {
RegExpZeroWidthMatch() { re.zeroWidthMatch(start, end) }
override predicate isCharacter() { any() }
override RegExpTerm getChild(int i) { none() }
override string getPrimaryQLClass() { result = "RegExpZeroWidthMatch" }
}
/**
* A zero-width lookahead or lookbehind assertion.
*
* Examples:
*
* ```
* (?=\w)
* (?!\n)
* (?<=\.)
* (?<!\\)
* ```
*/
class RegExpSubPattern extends RegExpZeroWidthMatch {
RegExpSubPattern() { not re.emptyGroup(start, end) }
}
/**
* A zero-width lookahead assertion.
*
* Examples:
*
* ```
* (?=\w)
* (?!\n)
* ```
*/
abstract class RegExpLookahead extends RegExpSubPattern { }
/**
* A positive-lookahead assertion.
*
* Examples:
*
* ```
* (?=\w)
* ```
*/
class RegExpPositiveLookahead extends RegExpLookahead {
RegExpPositiveLookahead() { re.positiveLookaheadAssertionGroup(start, end) }
override string getPrimaryQLClass() { result = "RegExpPositiveLookahead" }
}
/**
* A negative-lookahead assertion.
*
* Examples:
*
* ```
* (?!\n)
* ```
*/
class RegExpNegativeLookahead extends RegExpLookahead {
RegExpNegativeLookahead() { re.negativeLookaheadAssertionGroup(start, end) }
override string getPrimaryQLClass() { result = "RegExpNegativeLookahead" }
}
/**
* A zero-width lookbehind assertion.
*
* Examples:
*
* ```
* (?<=\.)
* (?<!\\)
* ```
*/
abstract class RegExpLookbehind extends RegExpSubPattern { }
/**
* A positive-lookbehind assertion.
*
* Examples:
*
* ```
* (?<=\.)
* ```
*/
class RegExpPositiveLookbehind extends RegExpLookbehind {
RegExpPositiveLookbehind() { re.positiveLookbehindAssertionGroup(start, end) }
override string getPrimaryQLClass() { result = "RegExpPositiveLookbehind" }
}
/**
* A negative-lookbehind assertion.
*
* Examples:
*
* ```
* (?<!\\)
* ```
*/
class RegExpNegativeLookbehind extends RegExpLookbehind {
RegExpNegativeLookbehind() { re.negativeLookbehindAssertionGroup(start, end) }
override string getPrimaryQLClass() { result = "RegExpNegativeLookbehind" }
}
/**
* A back reference, that is, a term of the form `\i` or `\k<name>`
* in a regular expression.
*
* Examples:
*
* ```
* \1
* (?P=quote)
* ```
*/
class RegExpBackRef extends RegExpTerm, TRegExpBackRef {
RegExpBackRef() { this = TRegExpBackRef(re, start, end) }
/**
* Gets the number of the capture group this back reference refers to, if any.
*/
int getNumber() { result = re.getBackrefNumber(start, end) }
/**
* Gets the name of the capture group this back reference refers to, if any.
*/
string getName() { result = re.getBackrefName(start, end) }
/** Gets the capture group this back reference refers to. */
RegExpGroup getGroup() {
result.getLiteral() = this.getLiteral() and
(
result.getNumber() = this.getNumber() or
result.getName() = this.getName()
)
}
override RegExpTerm getChild(int i) { none() }
override string getPrimaryQLClass() { result = "RegExpBackRef" }
}
/** Gets the parse tree resulting from parsing `re`, if such has been constructed. */
RegExpTerm getParsedRegExp(StrConst re) { result.getRegex() = re and result.isRootTerm() }

222
python/ql/src/semmle/python/SSA.qll
View File

@@ -1,222 +0,0 @@
/** SSA library */
import python
/**
* A single static assignment variable.
* An SSA variable is a variable which is only assigned once (statically).
* SSA variables can be defined as normal variables or by a phi node which can occur at joins in the flow graph.
* Definitions without uses do not have a SSA variable.
*/
class SsaVariable extends @py_ssa_var {
SsaVariable() { py_ssa_var(this, _) }
/** Gets the source variable */
Variable getVariable() { py_ssa_var(this, result) }
/** Gets a use of this variable */
ControlFlowNode getAUse() { py_ssa_use(result, this) }
/** Gets the definition (which may be a deletion) of this SSA variable */
ControlFlowNode getDefinition() { py_ssa_defn(this, result) }
/**
* Gets an argument of the phi function defining this variable.
* This predicate uses the raw SSA form produced by the extractor.
* In general, you should use `getAPrunedPhiInput()` instead.
*/
SsaVariable getAPhiInput() { py_ssa_phi(this, result) }
/**
* Gets the edge(s) (result->this.getDefinition()) on which the SSA variable 'input' defines this SSA variable.
* For each incoming edge `X->B`, where `B` is the basic block containing this phi-node, only one of the input SSA variables
* for this phi-node is live. This predicate returns the predecessor block such that the variable 'input'
* is the live variable on the edge result->B.
*/
BasicBlock getPredecessorBlockForPhiArgument(SsaVariable input) {
input = this.getAPhiInput() and
result = this.getAPredecessorBlockForPhi() and
input.getDefinition().getBasicBlock().dominates(result) and
/*
* Beware the case where an SSA variable that is an input on one edge dominates another edge.
* Consider (in SSA form):
* x0 = 0
* if cond:
* x1 = 1
* x2 = phi(x0, x1)
* use(x2)
*
* The definition of x0 dominates the exit from the block x1=1, even though it does not reach it.
* Hence we need to check that no other definition dominates the edge and actually reaches it.
* Note that if a dominates c and b dominates c, then either a dominates b or vice-versa.
*/
not exists(SsaVariable other, BasicBlock other_def |
not other = input and
other = this.getAPhiInput() and
other_def = other.getDefinition().getBasicBlock()
|
other_def.dominates(result) and
input.getDefinition().getBasicBlock().strictlyDominates(other_def)
)
}
/** Gets an argument of the phi function defining this variable, pruned of unlikely edges. */
SsaVariable getAPrunedPhiInput() {
result = this.getAPhiInput() and
exists(BasicBlock incoming | incoming = this.getPredecessorBlockForPhiArgument(result) |
not incoming.getLastNode().(RaisingNode).unlikelySuccessor(this.getDefinition())
)
}
/** Gets a variable that ultimately defines this variable and is not itself defined by another variable */
SsaVariable getAnUltimateDefinition() {
result = this and not exists(this.getAPhiInput())
or
result = this.getAPhiInput().getAnUltimateDefinition()
}
/** Gets a textual representation of this element. */
string toString() { result = "SSA Variable " + this.getId() }
Location getLocation() { result = this.getDefinition().getLocation() }
/** Gets the id (name) of this variable */
string getId() { result = this.getVariable().getId() }
/** Gets the incoming edges for a Phi node. */
private BasicBlock getAPredecessorBlockForPhi() {
exists(getAPhiInput()) and
result.getASuccessor() = this.getDefinition().getBasicBlock()
}
/** Gets the incoming edges for a Phi node, pruned of unlikely edges. */
private BasicBlock getAPrunedPredecessorBlockForPhi() {
result = this.getAPredecessorBlockForPhi() and
not result.unlikelySuccessor(this.getDefinition().getBasicBlock())
}
/** Whether it is possible to reach a use of this variable without passing a definition */
predicate reachableWithoutDefinition() {
not exists(this.getDefinition()) and not py_ssa_phi(this, _)
or
exists(SsaVariable var | var = this.getAPhiInput() | var.reachableWithoutDefinition())
or
/*
* For phi-nodes, there must be a corresponding phi-input for each control-flow
* predecessor. Otherwise, the variable will be undefined on that incoming edge.
* WARNING: the same phi-input may cover multiple predecessors, so this check
* cannot be done by counting.
*/
exists(BasicBlock incoming |
incoming = this.getAPredecessorBlockForPhi() and
not this.getAPhiInput().getDefinition().getBasicBlock().dominates(incoming)
)
}
/** Whether this variable may be undefined */
predicate maybeUndefined() {
not exists(this.getDefinition()) and not py_ssa_phi(this, _) and not this.implicitlyDefined()
or
this.getDefinition().isDelete()
or
exists(SsaVariable var | var = this.getAPrunedPhiInput() | var.maybeUndefined())
or
/*
* For phi-nodes, there must be a corresponding phi-input for each control-flow
* predecessor. Otherwise, the variable will be undefined on that incoming edge.
* WARNING: the same phi-input may cover multiple predecessors, so this check
* cannot be done by counting.
*/
exists(BasicBlock incoming |
reaches_end(incoming) and
incoming = this.getAPrunedPredecessorBlockForPhi() and
not this.getAPhiInput().getDefinition().getBasicBlock().dominates(incoming)
)
}
private predicate implicitlyDefined() {
not exists(this.getDefinition()) and
not py_ssa_phi(this, _) and
exists(GlobalVariable var | this.getVariable() = var |
globallyDefinedName(var.getId())
or
var.getId() = "__path__" and var.getScope().(Module).isPackageInit()
)
}
/**
* Gets the global variable that is accessed if this local is undefined.
* Only applies to local variables in class scopes.
*/
GlobalVariable getFallbackGlobal() {
exists(LocalVariable local, Class cls | this.getVariable() = local |
local.getScope() = cls and
result.getScope() = cls.getScope() and
result.getId() = local.getId() and
not exists(this.getDefinition())
)
}
/*
* Whether this SSA variable is the first parameter of a method
* (regardless of whether it is actually called self or not)
*/
predicate isSelf() {
exists(Function func |
func.isMethod() and
this.getDefinition().getNode() = func.getArg(0)
)
}
}
private predicate reaches_end(BasicBlock b) {
not exits_early(b) and
(
/* Entry point */
not exists(BasicBlock prev | prev.getASuccessor() = b)
or
exists(BasicBlock prev | prev.getASuccessor() = b | reaches_end(prev))
)
}
private predicate exits_early(BasicBlock b) {
exists(FunctionObject f |
f.neverReturns() and
f.getACall().getBasicBlock() = b
)
}
private predicate gettext_installed() {
// Good enough (and fast) approximation
exists(Module m | m.getName() = "gettext")
}
private predicate builtin_constant(string name) {
exists(Object::builtin(name))
or
name = "WindowsError"
or
name = "_" and gettext_installed()
}
private predicate auto_name(string name) {
name = "__file__" or name = "__builtins__" or name = "__name__"
}
/** Whether this name is (almost) always defined, ie. it is a builtin or VM defined name */
predicate globallyDefinedName(string name) { builtin_constant(name) or auto_name(name) }
/** An SSA variable that is backed by a global variable */
class GlobalSsaVariable extends EssaVariable {
GlobalSsaVariable() { this.getSourceVariable() instanceof GlobalVariable }
GlobalVariable getVariable() { result = this.getSourceVariable() }
string getId() { result = this.getVariable().getId() }
override string toString() { result = "GSSA Variable " + this.getId() }
}

153
python/ql/src/semmle/python/Scope.qll
View File

@@ -1,153 +0,0 @@
import python
/**
* A Scope. A scope is the lexical extent over which all identifiers with the same name refer to the same variable.
* Modules, Classes and Functions are all Scopes. There are no other scopes.
* The scopes for expressions that create new scopes, lambdas and comprehensions, are handled by creating an anonymous Function.
*/
class Scope extends Scope_ {
Module getEnclosingModule() { result = this.getEnclosingScope().getEnclosingModule() }
/**
* This method will be deprecated in the next release. Please use `getEnclosingScope()` instead.
* The reason for this is to avoid confusion around use of `x.getScope+()` where `x` might be an
* `AstNode` or a `Variable`. Forcing the users to write `x.getScope().getEnclosingScope*()` ensures that
* the apparent semantics and the actual semantics coincide.
* [ Gets the scope enclosing this scope (modules have no enclosing scope) ]
*/
Scope getScope() { none() }
/** Gets the scope enclosing this scope (modules have no enclosing scope) */
Scope getEnclosingScope() { none() }
/** Gets the statements forming the body of this scope */
StmtList getBody() { none() }
/** Gets the nth statement of this scope */
Stmt getStmt(int n) { none() }
/** Gets a top-level statement in this scope */
Stmt getAStmt() { none() }
Location getLocation() { none() }
/** Gets the name of this scope */
string getName() { py_strs(result, this, 0) }
/** Gets the docstring for this scope */
StrConst getDocString() { result = this.getStmt(0).(ExprStmt).getValue() }
/** Gets the entry point into this Scope's control flow graph */
ControlFlowNode getEntryNode() { py_scope_flow(result, this, -1) }
/** Gets the non-explicit exit from this Scope's control flow graph */
ControlFlowNode getFallthroughNode() { py_scope_flow(result, this, 0) }
/** Gets the exit of this scope following from a return statement */
ControlFlowNode getReturnNode() { py_scope_flow(result, this, 2) }
/** Gets an exit from this Scope's control flow graph */
ControlFlowNode getAnExitNode() { exists(int i | py_scope_flow(result, this, i) and i >= 0) }
/**
* Gets an exit from this Scope's control flow graph,
* that does not result from an exception
*/
ControlFlowNode getANormalExit() {
result = this.getFallthroughNode()
or
result = this.getReturnNode()
}
/** Holds if this a top-level (non-nested) class or function */
predicate isTopLevel() { this.getEnclosingModule() = this.getEnclosingScope() }
/** Holds if this scope is deemed to be public */
predicate isPublic() {
/* Not inside a function */
not this.getEnclosingScope() instanceof Function and
/* Not implicitly private */
this.getName().charAt(0) != "_" and
(
this instanceof Module
or
exists(Module m | m = this.getEnclosingScope() and m.isPublic() |
/* If the module has an __all__, is this in it */
not exists(m.getAnExport())
or
m.getAnExport() = this.getName()
)
or
exists(Class c | c = this.getEnclosingScope() |
this instanceof Function and
c.isPublic()
)
)
}
predicate contains(AstNode a) {
this.getBody().contains(a)
or
exists(Scope inner | inner.getEnclosingScope() = this | inner.contains(a))
}
/**
* Holds if this scope can be expected to execute before `other`.
* Modules precede functions and methods in those modules
* `__init__` precedes other methods. `__enter__` precedes `__exit__`.
* NOTE that this is context-insensitive, so a module "precedes" a function
* in that module, even if that function is called from the module scope.
*/
predicate precedes(Scope other) {
exists(Function f, string name | f = other and name = f.getName() |
if f.isMethod()
then
// The __init__ method is preceded by the enclosing module
this = f.getEnclosingModule() and name = "__init__"
or
exists(Class c, string pred_name |
// __init__ -> __enter__ -> __exit__
// __init__ -> other-methods
f.getScope() = c and
(
pred_name = "__init__" and not name = "__init__" and not name = "__exit__"
or
pred_name = "__enter__" and name = "__exit__"
)
|
this.getScope() = c and
pred_name = this.(Function).getName()
or
not exists(Function pre_func |
pre_func.getName() = pred_name and
pre_func.getScope() = c
) and
this = other.getEnclosingModule()
)
else
// Normal functions are preceded by the enclosing module
this = f.getEnclosingModule()
)
}
/**
* Gets the evaluation scope for code in this (lexical) scope.
* This is usually the scope itself, but may be an enclosing scope.
* Notably, for list comprehensions in Python 2.
*/
Scope getEvaluatingScope() { result = this }
/**
* Holds if this scope is in the source archive,
* that is it is part of the code specified, not library code
*/
predicate inSource() { exists(this.getEnclosingModule().getFile().getRelativePath()) }
Stmt getLastStatement() { result = this.getBody().getLastItem().getLastStatement() }
/** Whether this contains `inner` syntactically and `inner` has the same scope as `this` */
predicate containsInScope(AstNode inner) {
this.getBody().contains(inner) and
this = inner.getScope()
}
}

90
python/ql/src/semmle/python/SelfAttribute.qll
View File

@@ -1,90 +0,0 @@
/**
* Utilities to support queries about instance attribute accesses of
* the form `self.attr`.
*/
import python
private import semmle.python.pointsto.Filters
/**
* An attribute access where the left hand side of the attribute expression
* is `self`.
*/
class SelfAttribute extends Attribute {
SelfAttribute() { self_attribute(this, _) }
Class getClass() { self_attribute(this, result) }
}
/** Whether variable 'self' is the self variable in method 'method' */
private predicate self_variable(Function method, Variable self) {
self.isParameter() and
method.isMethod() and
method.getArg(0).asName() = self.getAnAccess()
}
/** Whether attribute is an access of the form `self.attr` in the body of the class 'cls' */
private predicate self_attribute(Attribute attr, Class cls) {
exists(Function f, Variable self | self_variable(f, self) |
self.getAnAccess() = attr.getObject() and
cls = f.getScope+()
)
}
/** Helper class for UndefinedClassAttribute.ql &amp; MaybeUndefinedClassAttribute.ql */
class SelfAttributeRead extends SelfAttribute {
SelfAttributeRead() {
this.getCtx() instanceof Load and
// Be stricter for loads.
// We want to generous as to what is defined (i.e. stores),
// but strict as to what needs to be defined (i.e. loads).
exists(ClassObject cls, FunctionObject func | cls.declaredAttribute(_) = func |
func.getFunction() = this.getScope() and
cls.getPyClass() = this.getClass()
)
}
predicate guardedByHasattr() {
exists(Variable var, ControlFlowNode n |
var.getAUse() = this.getObject().getAFlowNode() and
hasattr(n, var.getAUse(), this.getName()) and
n.strictlyDominates(this.getAFlowNode())
)
}
pragma[noinline]
predicate locallyDefined() {
exists(SelfAttributeStore store |
this.getName() = store.getName() and
this.getScope() = store.getScope()
|
store.getAFlowNode().strictlyDominates(this.getAFlowNode())
)
}
}
class SelfAttributeStore extends SelfAttribute {
SelfAttributeStore() { this.getCtx() instanceof Store }
Expr getAssignedValue() { exists(Assign a | a.getATarget() = this | result = a.getValue()) }
}
private predicate attr_assigned_in_method_arg_n(FunctionObject method, string name, int n) {
exists(SsaVariable param |
method.getFunction().getArg(n).asName() = param.getDefinition().getNode()
|
exists(AttrNode attr |
attr.getObject(name) = param.getAUse() and
attr.isStore()
)
or
exists(CallNode call, FunctionObject callee, int m |
callee.getArgumentForCall(call, m) = param.getAUse() and
attr_assigned_in_method_arg_n(callee, name, m)
)
)
}
predicate attribute_assigned_in_method(FunctionObject method, string name) {
attr_assigned_in_method_arg_n(method, name, 0)
}

117
python/ql/src/semmle/python/SpecialMethods.qll
View File

@@ -1,117 +0,0 @@
/**
* Provides support for special methods.
* This is done in two steps:
* - A subset of `ControlFlowNode`s are labelled as potentially corresponding to
* a special method call (by being an instance of `SpecialMethod::Potential`).
* - A subset of the potential special method calls are labelled as being actual
* special method calls (`SpecialMethodCallNode`) if the appropriate method is defined.
* Extend `SpecialMethod::Potential` to capture more cases.
*/
import python
/** A control flow node which might correspond to a special method call. */
class PotentialSpecialMethodCallNode extends ControlFlowNode {
PotentialSpecialMethodCallNode() { this instanceof SpecialMethod::Potential }
}
/**
* Machinery for detecting special method calls.
* Extend `SpecialMethod::Potential` to capture more cases.
*/
module SpecialMethod {
/** A control flow node which might correspond to a special method call. */
abstract class Potential extends ControlFlowNode {
/** Gets the name of the method that would be called. */
abstract string getSpecialMethodName();
/** Gets the control flow node that would be passed as the specified argument. */
abstract ControlFlowNode getArg(int n);
/**
* Gets the control flow node corresponding to the instance
* that would define the special method.
*/
ControlFlowNode getSelf() { result = this.getArg(0) }
}
/** A binary expression node that might correspond to a special method call. */
class SpecialBinOp extends Potential, BinaryExprNode {
Operator operator;
SpecialBinOp() { this.getOp() = operator }
override string getSpecialMethodName() { result = operator.getSpecialMethodName() }
override ControlFlowNode getArg(int n) {
n = 0 and result = this.getLeft()
or
n = 1 and result = this.getRight()
}
}
/** A subscript expression node that might correspond to a special method call. */
abstract class SpecialSubscript extends Potential, SubscriptNode {
override ControlFlowNode getArg(int n) {
n = 0 and result = this.getObject()
or
n = 1 and result = this.getIndex()
}
}
/** A subscript expression node that might correspond to a call to __getitem__. */
class SpecialGetItem extends SpecialSubscript {
SpecialGetItem() { this.isLoad() }
override string getSpecialMethodName() { result = "__getitem__" }
}
/** A subscript expression node that might correspond to a call to __setitem__. */
class SpecialSetItem extends SpecialSubscript {
SpecialSetItem() { this.isStore() }
override string getSpecialMethodName() { result = "__setitem__" }
override ControlFlowNode getArg(int n) {
n = 0 and result = this.getObject()
or
n = 1 and result = this.getIndex()
or
n = 2 and result = this.getValueNode()
}
private ControlFlowNode getValueNode() {
exists(AssignStmt a |
a.getATarget() = this.getNode() and
result.getNode() = a.getValue()
)
or
exists(AugAssign a |
a.getTarget() = this.getNode() and
result.getNode() = a.getValue()
)
}
}
/** A subscript expression node that might correspond to a call to __delitem__. */
class SpecialDelItem extends SpecialSubscript {
SpecialDelItem() { this.isDelete() }
override string getSpecialMethodName() { result = "__delitem__" }
}
}
/** A control flow node corresponding to a special method call. */
class SpecialMethodCallNode extends PotentialSpecialMethodCallNode {
Value resolvedSpecialMethod;
SpecialMethodCallNode() {
exists(SpecialMethod::Potential pot |
this.(SpecialMethod::Potential) = pot and
pot.getSelf().pointsTo().getClass().lookup(pot.getSpecialMethodName()) = resolvedSpecialMethod
)
}
/** The method that is called. */
Value getResolvedSpecialMethod() { result = resolvedSpecialMethod }
}

437
python/ql/src/semmle/python/Stmts.qll
View File

@@ -1,437 +0,0 @@
import python
/** A statement */
class Stmt extends Stmt_, AstNode {
/** Gets the scope immediately enclosing this statement */
override Scope getScope() { py_scopes(this, result) }
override string toString() { result = "Stmt" }
/** Gets the module enclosing this statement */
Module getEnclosingModule() { result = this.getScope().getEnclosingModule() }
override Location getLocation() { result = Stmt_.super.getLocation() }
/** Gets an immediate (non-nested) sub-expression of this statement */
Expr getASubExpression() { none() }
/** Gets an immediate (non-nested) sub-statement of this statement */
Stmt getASubStatement() { none() }
override AstNode getAChildNode() {
result = this.getASubExpression()
or
result = this.getASubStatement()
}
private ControlFlowNode possibleEntryNode() {
result.getNode() = this or
this.containsInScope(result.getNode())
}
/**
* Gets a control flow node for an entry into this statement.
*/
ControlFlowNode getAnEntryNode() {
result = this.possibleEntryNode() and
exists(ControlFlowNode pred |
pred.getASuccessor() = result and
not pred = this.possibleEntryNode()
)
}
/** Holds if this statement cannot be reached */
predicate isUnreachable() {
not exists(this.getAnEntryNode())
or
exists(If ifstmt |
ifstmt.getTest().(ImmutableLiteral).booleanValue() = false and ifstmt.getBody().contains(this)
or
ifstmt.getTest().(ImmutableLiteral).booleanValue() = true and
ifstmt.getOrelse().contains(this)
)
or
exists(While whilestmt |
whilestmt.getTest().(ImmutableLiteral).booleanValue() = false and
whilestmt.getBody().contains(this)
)
}
/**
* Gets the final statement in this statement, ordered by location.
* Will be this statement if not a compound statement.
*/
Stmt getLastStatement() { result = this }
}
/** A statement that includes a binding (except imports) */
class Assign extends Assign_ {
/** Use ControlFlowNodes and SsaVariables for data-flow analysis. */
predicate defines(Variable v) { this.getATarget().defines(v) }
override Expr getASubExpression() {
result = this.getATarget() or
result = this.getValue()
}
override Stmt getASubStatement() { none() }
}
/** An assignment statement */
class AssignStmt extends Assign {
/* syntax: Expr, ... = Expr */
AssignStmt() { not this instanceof FunctionDef and not this instanceof ClassDef }
override string toString() { result = "AssignStmt" }
}
/** An augmented assignment statement, such as `x += y` */
class AugAssign extends AugAssign_ {
/* syntax: Expr += Expr */
override Expr getASubExpression() { result = this.getOperation() }
/**
* Gets the target of this augmented assignment statement.
* That is, the `a` in `a += b`.
*/
Expr getTarget() { result = this.getOperation().(BinaryExpr).getLeft() }
/**
* Gets the value of this augmented assignment statement.
* That is, the `b` in `a += b`.
*/
Expr getValue() { result = this.getOperation().(BinaryExpr).getRight() }
override Stmt getASubStatement() { none() }
}
/** An annotated assignment statement, such as `x: int = 0` */
class AnnAssign extends AnnAssign_ {
/* syntax: Expr: Expr = Expr */
override Expr getASubExpression() {
result = this.getAnnotation() or
result = this.getTarget() or
result = this.getValue()
}
override Stmt getASubStatement() { none() }
/** Holds if the value of the annotation of this assignment is stored at runtime. */
predicate isStored() {
not this.getScope() instanceof Function and
exists(Name n |
n = this.getTarget() and
not n.isParenthesized()
)
}
}
/** An exec statement */
class Exec extends Exec_ {
/* syntax: exec Expr */
override Expr getASubExpression() {
result = this.getBody() or
result = this.getGlobals() or
result = this.getLocals()
}
override Stmt getASubStatement() { none() }
}
/** An except statement (part of a `try` statement), such as `except IOError as err:` */
class ExceptStmt extends ExceptStmt_ {
/* syntax: except Expr [ as Expr ]: */
/** Gets the immediately enclosing try statement */
Try getTry() { result.getAHandler() = this }
override Expr getASubExpression() {
result = this.getName()
or
result = this.getType()
}
override Stmt getASubStatement() { result = this.getAStmt() }
override Stmt getLastStatement() { result = this.getBody().getLastItem().getLastStatement() }
}
/** An assert statement, such as `assert a == b, "A is not equal to b"` */
class Assert extends Assert_ {
/* syntax: assert Expr [, Expr] */
override Expr getASubExpression() { result = this.getMsg() or result = this.getTest() }
override Stmt getASubStatement() { none() }
}
/** A break statement */
class Break extends Break_ {
/* syntax: assert Expr [, Expr] */
override Expr getASubExpression() { none() }
override Stmt getASubStatement() { none() }
}
/** A continue statement */
class Continue extends Continue_ {
/* syntax: continue */
override Expr getASubExpression() { none() }
override Stmt getASubStatement() { none() }
}
/** A delete statement, such as `del x[-1]` */
class Delete extends Delete_ {
/* syntax: del Expr, ... */
override Expr getASubExpression() { result = this.getATarget() }
override Stmt getASubStatement() { none() }
}
/** An expression statement, such as `len(x)` or `yield y` */
class ExprStmt extends ExprStmt_ {
/* syntax: Expr */
override Expr getASubExpression() { result = this.getValue() }
override Stmt getASubStatement() { none() }
}
/** A for statement, such as `for x in y: print(x)` */
class For extends For_ {
/* syntax: for varname in Expr: ... */
override Stmt getASubStatement() {
result = this.getAStmt() or
result = this.getAnOrelse()
}
override Expr getASubExpression() {
result = this.getTarget() or
result = this.getIter()
}
override Stmt getLastStatement() { result = this.getBody().getLastItem().getLastStatement() }
}
/** A global statement, such as `global var` */
class Global extends Global_ {
/* syntax: global varname */
override Expr getASubExpression() { none() }
override Stmt getASubStatement() { none() }
}
/** An if statement, such as `if eggs: print("spam")` */
class If extends If_ {
/* syntax: if Expr: ... */
override Stmt getASubStatement() {
result = this.getAStmt() or
result = this.getAnOrelse()
}
override Expr getASubExpression() { result = this.getTest() }
/** Whether this if statement takes the form `if __name__ == "__main__":` */
predicate isNameEqMain() {
exists(StrConst m, Name n, Compare c |
this.getTest() = c and
c.getOp(0) instanceof Eq and
(
c.getLeft() = n and c.getComparator(0) = m
or
c.getLeft() = m and c.getComparator(0) = n
) and
n.getId() = "__name__" and
m.getText() = "__main__"
)
}
/** Whether this if statement starts with the keyword `elif` */
predicate isElif() {
/*
* The Python parser turns all elif chains into nested if-else statements.
* An `elif` can be identified as it is the first statement in an `else` block
* and it is not indented relative to its parent `if`.
*/
exists(If i |
i.getOrelse(0) = this and
this.getLocation().getStartColumn() = i.getLocation().getStartColumn()
)
}
/** Gets the `elif` branch of this `if`-statement, if present */
If getElif() {
result = this.getOrelse(0) and
result.isElif()
}
override Stmt getLastStatement() {
result = this.getOrelse().getLastItem().getLastStatement()
or
not exists(this.getOrelse()) and
result = this.getBody().getLastItem().getLastStatement()
}
}
/** A nonlocal statement, such as `nonlocal var` */
class Nonlocal extends Nonlocal_ {
/* syntax: nonlocal varname */
override Stmt getASubStatement() { none() }
override Expr getASubExpression() { none() }
Variable getAVariable() {
result.getScope() = this.getScope() and
result.getId() = this.getAName()
}
}
/** A pass statement */
class Pass extends Pass_ {
/* syntax: pass */
override Stmt getASubStatement() { none() }
override Expr getASubExpression() { none() }
}
/** A print statement (Python 2 only), such as `print 0` */
class Print extends Print_ {
/* syntax: print Expr, ... */
override Stmt getASubStatement() { none() }
override Expr getASubExpression() {
result = this.getAValue() or
result = this.getDest()
}
}
/** A raise statement, such as `raise CompletelyDifferentException()` */
class Raise extends Raise_ {
/* syntax: raise Expr */
override Stmt getASubStatement() { none() }
override Expr getASubExpression() { py_exprs(result, _, this, _) }
/**
* The expression immediately following the `raise`, this is the
* exception raised, but not accounting for tuples in Python 2.
*/
Expr getException() {
result = this.getType()
or
result = this.getExc()
}
/** The exception raised, accounting for tuples in Python 2. */
Expr getRaised() {
exists(Expr raw | raw = this.getException() |
if not major_version() = 2 or not exists(raw.(Tuple).getAnElt())
then result = raw
else
/* In Python 2 raising a tuple will result in the first element of the tuple being raised. */
result = raw.(Tuple).getElt(0)
)
}
}
/** A return statement, such as return None */
class Return extends Return_ {
/* syntax: return Expr */
override Stmt getASubStatement() { none() }
override Expr getASubExpression() { result = this.getValue() }
}
/** A try statement */
class Try extends Try_ {
/* syntax: try: ... */
override Expr getASubExpression() { none() }
override Stmt getASubStatement() {
result = this.getAHandler() or
result = this.getAStmt() or
result = this.getAFinalstmt() or
result = this.getAnOrelse()
}
override ExceptStmt getHandler(int i) { result = Try_.super.getHandler(i) }
/** Gets an exception handler of this try statement. */
override ExceptStmt getAHandler() { result = Try_.super.getAHandler() }
override Stmt getLastStatement() {
result = this.getFinalbody().getLastItem().getLastStatement()
or
not exists(this.getFinalbody()) and
result = this.getOrelse().getLastItem().getLastStatement()
or
not exists(this.getFinalbody()) and
not exists(this.getOrelse()) and
result = this.getHandlers().getLastItem().getLastStatement()
or
not exists(this.getFinalbody()) and
not exists(this.getOrelse()) and
not exists(this.getHandlers()) and
result = this.getBody().getLastItem().getLastStatement()
}
}
/** A while statement, such as `while parrot_resting():` */
class While extends While_ {
/* syntax: while Expr: ... */
override Expr getASubExpression() { result = this.getTest() }
override Stmt getASubStatement() {
result = this.getAStmt() or
result = this.getAnOrelse()
}
override Stmt getLastStatement() {
result = this.getOrelse().getLastItem().getLastStatement()
or
not exists(this.getOrelse()) and
result = this.getBody().getLastItem().getLastStatement()
}
}
/** A with statement such as `with f as open("file"): text = f.read()` */
class With extends With_ {
/* syntax: with Expr as varname: ... */
override Expr getASubExpression() {
result = this.getContextExpr() or
result = this.getOptionalVars()
}
override Stmt getASubStatement() { result = this.getAStmt() }
override Stmt getLastStatement() { result = this.getBody().getLastItem().getLastStatement() }
}
/** A plain text used in a template is wrapped in a TemplateWrite statement */
class TemplateWrite extends TemplateWrite_ {
override Expr getASubExpression() { result = this.getValue() }
override Stmt getASubStatement() { none() }
}
/** An asynchronous `for` statement, such as `async for varname in Expr: ...` */
class AsyncFor extends For {
/* syntax: async for varname in Expr: ... */
AsyncFor() { this.isAsync() }
}
/** An asynchronous `with` statement, such as `async with varname as Expr: ...` */
class AsyncWith extends With {
/* syntax: async with Expr as varname: ... */
AsyncWith() { this.isAsync() }
}
/** A list of statements */
class StmtList extends StmtList_ {
/** Holds if this list of statements contains the AST node `a` */
predicate contains(AstNode a) {
exists(Stmt item | item = this.getAnItem() | item = a or item.contains(a))
}
/** Gets the last item in this list of statements, if any. */
Stmt getLastItem() { result = this.getItem(max(int i | exists(this.getItem(i)))) }
}

Some files were not shown because too many files have changed in this diff Show More