hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2025-12-17 01:03:14 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into js/graph-export

Browse Source
This commit is contained in:
Asger F
2024-04-16 20:23:33 +02:00
parent 844b29b637 5c50ed036c
commit be64daf265
1236 changed files with 37405 additions and 42510 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
python/downgrades/BUILD.bazel
View File

@@ -1,4 +1,4 @@
load("@rules_pkg//:mappings.bzl", "pkg_files", "strip_prefix")
load("@rules_pkg//pkg:mappings.bzl", "pkg_files", "strip_prefix")
pkg_files(
name = "downgrades",

1
python/extractor/BUILD.bazel
View File

@@ -5,7 +5,6 @@ py_binary(
srcs = [
"make_zips.py",
"python_tracer.py",
"unparse.py",
],
data = [
"LICENSE-PSF.md",

6
python/extractor/buildtools/version.py
View File

@@ -18,12 +18,12 @@ else:
WIN = sys.platform == "win32"
if WIN:
if WIN and "CODEQL_EXTRACTOR_PYTHON_OPTION_PYTHON_EXECUTABLE_NAME" not in os.environ:
# installing `py` launcher is optional when installing Python on windows, so it's
# possible that the user did not install it, see
# https://github.com/github/codeql-cli-binaries/issues/125#issuecomment-1157429430
# so we check whether it has been installed. Newer versions have a `--list` option,
# so we check whether it has been installed, and we check only if the "python_executable_name"
# extractor option has not been specified. Newer versions have a `--list` option,
# but that has only been mentioned in the docs since 3.9, so to not risk it not
# working on potential older versions, we'll just use `py --version` which forwards
# the `--version` argument to the default python executable.

41
python/extractor/cli-integration-test/force-enable-library-extraction/disabled-test.sh
View File

@@ -1,41 +0,0 @@
#!/bin/bash
set -Eeuo pipefail # see https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail/
set -x
CODEQL=${CODEQL:-codeql}
SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd "$SCRIPTDIR"
# start on clean slate
rm -rf dbs
mkdir dbs
cd "$SCRIPTDIR"
export CODEQL_EXTRACTOR_PYTHON_FORCE_ENABLE_LIBRARY_EXTRACTION_UNTIL_2_17_0=
$CODEQL database create dbs/normal --language python --source-root repo_dir/
export CODEQL_EXTRACTOR_PYTHON_FORCE_ENABLE_LIBRARY_EXTRACTION_UNTIL_2_17_0=1
$CODEQL database create dbs/with-lib-extraction --language python --source-root repo_dir/
# ---
set +x
EXTRACTED_NORMAL=$(unzip -l dbs/normal/src.zip | wc -l)
EXTRACTED_WITH_LIB_EXTRACTION=$(unzip -l dbs/with-lib-extraction/src.zip | wc -l)
exitcode=0
echo "EXTRACTED_NORMAL=$EXTRACTED_NORMAL"
echo "EXTRACTED_WITH_LIB_EXTRACTION=$EXTRACTED_WITH_LIB_EXTRACTION"
if [[ ! $EXTRACTED_WITH_LIB_EXTRACTION -gt $EXTRACTED_NORMAL ]]; then
echo "ERROR: EXTRACTED_WITH_LIB_EXTRACTION not greater than EXTRACTED_NORMAL"
exitcode=1
fi
exit $exitcode

3
python/extractor/cli-integration-test/force-enable-library-extraction/repo_dir/foo.py
View File

@@ -1,3 +0,0 @@
import pip
print(42)

79
python/extractor/cli-integration-test/ignore-venv/disabled-test.sh
View File

@@ -1,79 +0,0 @@
#!/bin/bash
set -Eeuo pipefail # see https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail/
set -x
CODEQL=${CODEQL:-codeql}
SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd "$SCRIPTDIR"
# start on clean slate
rm -rf dbs repo_dir/venv*
mkdir dbs
# set up venvs
cd repo_dir
python3 -m venv venv
venv/bin/pip install flask
python3 -m venv venv2
cd "$SCRIPTDIR"
# In 2.16.0 we stop extracting libraries by default, so to test this functionality we
# need to force enable it. Once we release 2.17.0 and turn off library extraction for
# good, we can remove the part of this test ensuring that dependencies in an active
# venv are still extracted (since that will no longer be the case).
export CODEQL_EXTRACTOR_PYTHON_FORCE_ENABLE_LIBRARY_EXTRACTION_UNTIL_2_17_0=1
# Create DBs with venv2 active (that does not have flask installed)
source repo_dir/venv2/bin/activate
export CODEQL_EXTRACTOR_PYTHON_DISABLE_AUTOMATIC_VENV_EXCLUDE=
$CODEQL database create dbs/normal --language python --source-root repo_dir/
export CODEQL_EXTRACTOR_PYTHON_DISABLE_AUTOMATIC_VENV_EXCLUDE=1
$CODEQL database create dbs/no-venv-ignore --language python --source-root repo_dir/
# Create DB with venv active that has flask installed. We want to ensure that we're
# still able to resolve imports to flask, but don't want to extract EVERYTHING from
# within the venv. Important note is that the test-file in the repo_dir actually imports
# flask :D
source repo_dir/venv/bin/activate
export CODEQL_EXTRACTOR_PYTHON_DISABLE_AUTOMATIC_VENV_EXCLUDE=
$CODEQL database create dbs/normal-with-flask-venv --language python --source-root repo_dir/
# ---
set +x
EXTRACTED_NORMAL=$(unzip -l dbs/normal/src.zip | wc -l)
EXTRACTED_NO_VENV_IGNORE=$(unzip -l dbs/no-venv-ignore/src.zip | wc -l)
EXTRACTED_ACTIVE_FLASK=$(unzip -l dbs/normal-with-flask-venv/src.zip | wc -l)
exitcode=0
echo "EXTRACTED_NORMAL=$EXTRACTED_NORMAL"
echo "EXTRACTED_NO_VENV_IGNORE=$EXTRACTED_NO_VENV_IGNORE"
echo "EXTRACTED_ACTIVE_FLASK=$EXTRACTED_ACTIVE_FLASK"
if [[ ! $EXTRACTED_NORMAL -lt $EXTRACTED_NO_VENV_IGNORE ]]; then
echo "ERROR: EXTRACTED_NORMAL not smaller EXTRACTED_NO_VENV_IGNORE"
exitcode=1
fi
if [[ ! $EXTRACTED_NORMAL -lt $EXTRACTED_ACTIVE_FLASK ]]; then
echo "ERROR: EXTRACTED_NORMAL not smaller EXTRACTED_ACTIVE_FLASK"
exitcode=1
fi
if [[ ! $EXTRACTED_ACTIVE_FLASK -lt $EXTRACTED_NO_VENV_IGNORE ]]; then
echo "ERROR: EXTRACTED_ACTIVE_FLASK not smaller EXTRACTED_NO_VENV_IGNORE"
exitcode=1
fi
exit $exitcode

49
python/extractor/cli-integration-test/ignore-venv/test.sh Executable file
View File

@@ -0,0 +1,49 @@
#!/bin/bash
set -Eeuo pipefail # see https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail/
set -x
CODEQL=${CODEQL:-codeql}
SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd "$SCRIPTDIR"
# start on clean slate
rm -rf dbs repo_dir/venv*
mkdir dbs
# set up venvs
cd repo_dir
# make venv with some package in it (so we show that our ignore logic is correct)
python3 -m venv venv
venv/bin/pip install flask
cd "$SCRIPTDIR"
export CODEQL_EXTRACTOR_PYTHON_DISABLE_AUTOMATIC_VENV_EXCLUDE=
$CODEQL database create dbs/normal --language python --source-root repo_dir/
export CODEQL_EXTRACTOR_PYTHON_DISABLE_AUTOMATIC_VENV_EXCLUDE=1
$CODEQL database create dbs/no-venv-ignore --language python --source-root repo_dir/
# ---
set +x
EXTRACTED_NORMAL=$(unzip -l dbs/normal/src.zip | wc -l)
EXTRACTED_NO_VENV_IGNORE=$(unzip -l dbs/no-venv-ignore/src.zip | wc -l)
exitcode=0
echo "EXTRACTED_NORMAL=$EXTRACTED_NORMAL"
echo "EXTRACTED_NO_VENV_IGNORE=$EXTRACTED_NO_VENV_IGNORE"
if [[ ! $EXTRACTED_NORMAL -lt $EXTRACTED_NO_VENV_IGNORE ]]; then
echo "ERROR: EXTRACTED_NORMAL not smaller EXTRACTED_NO_VENV_IGNORE"
exitcode=1
fi
exit $exitcode

1
python/extractor/licenses.md
View File

@@ -6,7 +6,6 @@
| `tsg-python/tree-sitter-python` | Y | MIT | Used in `tsg-python` to parse Python files |
| `tsg-python` | Y | MIT / Apache | This is our own creation, so are free to choose what license it is covered by. |
| `tree-sitter-graph` | N | MIT / Apache | Used in `tsg-python` to execute files written in the `tree-sitter-graph` language. |
| `unparse.py` | Y | PSF | Copied and adapted from `Tools/unparse.py` from the `cpython` source code, with attribution. |
| `imp.py` | Y | PSF | Copied and adapted from `Lib/imp.py` from the `cpython` source code, with attribution. |
| `semmle/data/*.trap` | Y | PSF | These files were derived from the C source code of the `cpython` project, and are used in our modelling of built-in objects. No attribution, currently. |
| `semmle/thrift/parse.py` | Y | Apache | Includes a grammar based on https://github.com/apache/thrift/blob/master/doc/specs/idl.md, with comment stating this attribution. |

5
python/extractor/make_zips.py
View File

@@ -8,7 +8,6 @@ import optparse
import compileall
from python_tracer import getzipfilename
from unparse import strip_comments_and_docstrings
# TO DO -- Add options to set destination directory and source directory
@@ -84,9 +83,7 @@ def write_source(zipped, root, name, extensions=[".py"]):
if ext not in extensions:
continue
path = os.path.join(dirpath, name)
temp = strip_comments_and_docstrings(path)
zipped.write(temp, os.path.relpath(path, root))
os.remove(temp)
zipped.write(path, os.path.relpath(path, root))
def main():
parser = optparse.OptionParser(usage = "usage: %prog [install-dir]")

11
python/extractor/tsg-python/Cargo.toml
View File

@@ -7,9 +7,14 @@ authors = ["Taus Brock-Nannestad <tausbn@github.com>"]
edition = "2018"
# When changing/updating these, the `Cargo.Bazel.lock` file has to be regenerated.
# Check out the documentation at https://bazelbuild.github.io/rules_rust/crate_universe.html#repinning--updating-dependencies
# for how to do so. The bazel repository for the tsg-python project is called `py_deps`,
# and instead of calling `bazel sync`, `./build --bazel sync` should be used instead, to always use the correct bazel version.
# Run `CARGO_BAZEL_REPIN=true CARGO_BAZEL_REPIN_ONLY=py_deps ./build --bazel sync --only=py_deps`
# in the `semmle-code` repository to do so.
# For more information, check out the documentation at
# https://bazelbuild.github.io/rules_rust/crate_universe.html#repinning--updating-dependencies
# In the future, the hope is to move this handling of the dependencies entirely into the `codeql` repository,
# but that depends on `rules_rust` being fully compatible with bzlmod, which they aren't yet
# (c.f. https://github.com/bazelbuild/rules_rust/issues/2452).
# Warning: The process takes >5min on my M1 mac, so do wait for a while.
[dependencies]
anyhow = "1.0"
regex = "1"

4
python/extractor/tsg-python/tsp/BUILD.bazel
View File

@@ -7,7 +7,7 @@ package(default_visibility = ["//visibility:public"])
# This will run the build script from the root of the workspace, and
# collect the outputs.
cargo_build_script(
name = "tsg-build",
name = "tsp-build",
srcs = ["bindings/rust/build.rs"],
data = glob([
"src/**",
@@ -32,7 +32,7 @@ rust_library(
proc_macro_deps = all_crate_deps(
proc_macro = True,
),
deps = [":tsg-build"] + all_crate_deps(
deps = [":tsp-build"] + all_crate_deps(
normal = True,
),
)

709
python/extractor/unparse.py
View File

@@ -1,709 +0,0 @@
#Copied Tools.unparse.py with modifications. Copyright PSF.
"Usage: unparse.py <path to source file>"
import sys
import ast
import tokenize
import io
import os
import shutil
# Large float and imaginary literals get turned into infinities in the AST.
# We unparse those infinities to INFSTR.
INFSTR = "1e" + repr(sys.float_info.max_10_exp + 1)
def interleave(inter, f, seq):
"""Call f on each item in seq, calling inter() in between.
"""
seq = iter(seq)
try:
f(next(seq))
except StopIteration:
pass
else:
for x in seq:
inter()
f(x)
class Unparser:
"""Methods in this class recursively traverse an AST and
output source code for the abstract syntax; original formatting
is disregarded. """
def __init__(self, tree, file = sys.stdout):
"""Unparser(tree, file=sys.stdout) -> None.
Print the source for tree to file."""
self.f = file
self._indent = 0
self.dispatch(tree)
print("", file=self.f)
self.f.flush()
def fill(self, text = ""):
"Indent a piece of text, according to the current indentation level"
self.f.write("\n"+" "*self._indent + text)
def write(self, text):
"Append a piece of text to the current line."
self.f.write(text)
def enter(self):
"Print ':', and increase the indentation."
self.write(":")
self._indent += 1
def leave(self):
"Decrease the indentation level."
self._indent -= 1
def dispatch(self, tree):
"Dispatcher function, dispatching tree type T to method _T."
if isinstance(tree, list):
for t in tree:
self.dispatch(t)
return
meth = getattr(self, "_"+tree.__class__.__name__)
meth(tree)
def remove_docstring(self, t):
if hasattr(t, "docstring"):
return
if not t.body:
return
if not isinstance(t.body[0], ast.Expr):
return
if not isinstance(t.body[0].value, ast.Str):
return
t.body = t.body[1:]
def add_pass(self, t):
if t.body:
#No pass needed
return
t.body = [ast.Pass()]
############### Unparsing methods ######################
# There should be one method per concrete grammar type #
# Constructors should be grouped by sum type. Ideally, #
# this would follow the order in the grammar, but #
# currently doesn't. #
########################################################
def _Module(self, tree):
self.remove_docstring(tree)
self.add_pass(tree)
for stmt in tree.body:
self.dispatch(stmt)
# stmt
def _Expr(self, tree):
self.fill()
self.dispatch(tree.value)
def _Import(self, t):
self.fill("import ")
interleave(lambda: self.write(", "), self.dispatch, t.names)
def _ImportFrom(self, t):
self.fill("from ")
self.write("." * t.level)
if t.module:
self.write(t.module)
self.write(" import ")
interleave(lambda: self.write(", "), self.dispatch, t.names)
def _Assign(self, t):
self.fill()
for target in t.targets:
self.dispatch(target)
self.write(" = ")
self.dispatch(t.value)
def _AugAssign(self, t):
self.fill()
self.dispatch(t.target)
self.write(" "+self.binop[t.op.__class__.__name__]+"= ")
self.dispatch(t.value)
def _AnnAssign(self, t):
self.fill()
if not t.simple and isinstance(t.target, ast.Name):
self.write('(')
self.dispatch(t.target)
if not t.simple and isinstance(t.target, ast.Name):
self.write(')')
self.write(": ")
self.dispatch(t.annotation)
if t.value:
self.write(" = ")
self.dispatch(t.value)
def _Return(self, t):
self.fill("return")
if t.value:
self.write(" ")
self.dispatch(t.value)
def _Pass(self, t):
self.fill("pass")
def _Break(self, t):
self.fill("break")
def _Continue(self, t):
self.fill("continue")
def _Delete(self, t):
self.fill("del ")
interleave(lambda: self.write(", "), self.dispatch, t.targets)
def _Assert(self, t):
self.fill("assert ")
self.dispatch(t.test)
if t.msg:
self.write(", ")
self.dispatch(t.msg)
def _Global(self, t):
self.fill("global ")
interleave(lambda: self.write(", "), self.write, t.names)
def _Nonlocal(self, t):
self.fill("nonlocal ")
interleave(lambda: self.write(", "), self.write, t.names)
def _Await(self, t):
self.write("(")
self.write("await")
if t.value:
self.write(" ")
self.dispatch(t.value)
self.write(")")
def _Yield(self, t):
self.write("(")
self.write("yield")
if t.value:
self.write(" ")
self.dispatch(t.value)
self.write(")")
def _YieldFrom(self, t):
self.write("(")
self.write("yield from")
if t.value:
self.write(" ")
self.dispatch(t.value)
self.write(")")
def _Raise(self, t):
self.fill("raise")
if not t.exc:
assert not t.cause
return
self.write(" ")
self.dispatch(t.exc)
if t.cause:
self.write(" from ")
self.dispatch(t.cause)
def _Try(self, t):
self.fill("try")
self.enter()
self.dispatch(t.body)
self.leave()
for ex in t.handlers:
self.dispatch(ex)
if t.orelse:
self.fill("else")
self.enter()
self.dispatch(t.orelse)
self.leave()
if t.finalbody:
self.fill("finally")
self.enter()
self.dispatch(t.finalbody)
self.leave()
def _ExceptHandler(self, t):
self.fill("except")
if t.type:
self.write(" ")
self.dispatch(t.type)
if t.name:
self.write(" as ")
self.write(t.name)
self.enter()
self.dispatch(t.body)
self.leave()
def _ClassDef(self, t):
self.write("\n")
for deco in t.decorator_list:
self.fill("@")
self.dispatch(deco)
self.fill("class "+t.name)
self.write("(")
comma = False
for e in t.bases:
if comma: self.write(", ")
else: comma = True
self.dispatch(e)
for e in t.keywords:
if comma: self.write(", ")
else: comma = True
self.dispatch(e)
self.write(")")
self.enter()
self.remove_docstring(t)
self.add_pass(t)
self.dispatch(t.body)
self.leave()
def _FunctionDef(self, t):
self.__FunctionDef_helper(t, "def")
def _AsyncFunctionDef(self, t):
self.__FunctionDef_helper(t, "async def")
def __FunctionDef_helper(self, t, fill_suffix):
self.write("\n")
for deco in t.decorator_list:
self.fill("@")
self.dispatch(deco)
def_str = fill_suffix+" "+t.name + "("
self.fill(def_str)
self.dispatch(t.args)
self.write(")")
if t.returns:
self.write(" -> ")
self.dispatch(t.returns)
self.enter()
self.remove_docstring(t)
self.add_pass(t)
self.dispatch(t.body)
self.leave()
def _For(self, t):
self.__For_helper("for ", t)
def _AsyncFor(self, t):
self.__For_helper("async for ", t)
def __For_helper(self, fill, t):
self.fill(fill)
self.dispatch(t.target)
self.write(" in ")
self.dispatch(t.iter)
self.enter()
self.dispatch(t.body)
self.leave()
if t.orelse:
self.fill("else")
self.enter()
self.dispatch(t.orelse)
self.leave()
def _If(self, t):
self.fill("if ")
self.dispatch(t.test)
self.enter()
self.dispatch(t.body)
self.leave()
# collapse nested ifs into equivalent elifs.
while (t.orelse and len(t.orelse) == 1 and
isinstance(t.orelse[0], ast.If)):
t = t.orelse[0]
self.fill("elif ")
self.dispatch(t.test)
self.enter()
self.dispatch(t.body)
self.leave()
# final else
if t.orelse:
self.fill("else")
self.enter()
self.dispatch(t.orelse)
self.leave()
def _While(self, t):
self.fill("while ")
self.dispatch(t.test)
self.enter()
self.dispatch(t.body)
self.leave()
if t.orelse:
self.fill("else")
self.enter()
self.dispatch(t.orelse)
self.leave()
def _With(self, t):
self.fill("with ")
interleave(lambda: self.write(", "), self.dispatch, t.items)
self.enter()
self.dispatch(t.body)
self.leave()
def _AsyncWith(self, t):
self.fill("async with ")
interleave(lambda: self.write(", "), self.dispatch, t.items)
self.enter()
self.dispatch(t.body)
self.leave()
# expr
def _Bytes(self, t):
self.write(repr(t.s))
def _Str(self, tree):
s = repr(tree.s).encode("ascii", errors="backslashreplace").decode("ascii")
self.write(s)
def _JoinedStr(self, t):
self.write("f")
string = io.StringIO()
self._fstring_JoinedStr(t, string.write)
self.write(repr(string.getvalue()))
def _FormattedValue(self, t):
self.write("f")
string = io.StringIO()
self._fstring_FormattedValue(t, string.write)
self.write(repr(string.getvalue()))
def _fstring_JoinedStr(self, t, write):
for value in t.values:
meth = getattr(self, "_fstring_" + type(value).__name__)
meth(value, write)
def _fstring_Str(self, t, write):
value = t.s.replace("{", "{{").replace("}", "}}")
write(value)
def _fstring_Constant(self, t, write):
assert isinstance(t.value, str)
value = t.value.replace("{", "{{").replace("}", "}}")
write(value)
def _fstring_FormattedValue(self, t, write):
write("{")
expr = io.StringIO()
Unparser(t.value, expr)
expr = expr.getvalue().rstrip("\n")
if expr.startswith("{"):
write(" ") # Separate pair of opening brackets as "{ {"
write(expr)
if t.conversion != -1:
conversion = chr(t.conversion)
assert conversion in "sra"
write("!%s" % conversion)
if t.format_spec:
write(":")
meth = getattr(self, "_fstring_" + type(t.format_spec).__name__)
meth(t.format_spec, write)
write("}")
def _Name(self, t):
self.write(t.id)
def _write_constant(self, value):
if isinstance(value, (float, complex)):
self.write(repr(value).replace("inf", INFSTR))
else:
self.write(repr(value))
def _Constant(self, t):
value = t.value
if isinstance(value, tuple):
self.write("(")
if len(value) == 1:
self._write_constant(value[0])
self.write(",")
else:
interleave(lambda: self.write(", "), self._write_constant, value)
self.write(")")
else:
self._write_constant(t.value)
def _NameConstant(self, t):
self.write(repr(t.value))
def _Num(self, t):
# Substitute overflowing decimal literal for AST infinities.
self.write(repr(t.n).replace("inf", INFSTR))
def _List(self, t):
self.write("[")
interleave(lambda: self.write(", "), self.dispatch, t.elts)
self.write("]")
def _ListComp(self, t):
self.write("[")
self.dispatch(t.elt)
for gen in t.generators:
self.dispatch(gen)
self.write("]")
def _GeneratorExp(self, t):
self.write("(")
self.dispatch(t.elt)
for gen in t.generators:
self.dispatch(gen)
self.write(")")
def _SetComp(self, t):
self.write("{")
self.dispatch(t.elt)
for gen in t.generators:
self.dispatch(gen)
self.write("}")
def _DictComp(self, t):
self.write("{")
self.dispatch(t.key)
self.write(": ")
self.dispatch(t.value)
for gen in t.generators:
self.dispatch(gen)
self.write("}")
def _comprehension(self, t):
if hasattr(t, "is_async") and t.is_async:
self.write(" async for ")
else:
self.write(" for ")
self.dispatch(t.target)
self.write(" in ")
self.dispatch(t.iter)
for if_clause in t.ifs:
self.write(" if ")
self.dispatch(if_clause)
def _IfExp(self, t):
self.write("(")
self.dispatch(t.body)
self.write(" if ")
self.dispatch(t.test)
self.write(" else ")
self.dispatch(t.orelse)
self.write(")")
def _Set(self, t):
assert(t.elts) # should be at least one element
self.write("{")
interleave(lambda: self.write(", "), self.dispatch, t.elts)
self.write("}")
def _Dict(self, t):
self.write("{")
def write_key_value_pair(k, v):
self.dispatch(k)
self.write(": ")
self.dispatch(v)
def write_item(item):
k, v = item
if k is None:
# for dictionary unpacking operator in dicts {**{'y': 2}}
# see PEP 448 for details
self.write("**")
self.dispatch(v)
else:
write_key_value_pair(k, v)
interleave(lambda: self.write(", "), write_item, zip(t.keys, t.values))
self.write("}")
def _Tuple(self, t):
self.write("(")
if len(t.elts) == 1:
elt = t.elts[0]
self.dispatch(elt)
self.write(",")
else:
interleave(lambda: self.write(", "), self.dispatch, t.elts)
self.write(")")
unop = {"Invert":"~", "Not": "not", "UAdd":"+", "USub":"-"}
def _UnaryOp(self, t):
self.write("(")
self.write(self.unop[t.op.__class__.__name__])
self.write(" ")
self.dispatch(t.operand)
self.write(")")
binop = { "Add":"+", "Sub":"-", "Mult":"*", "MatMult":"@", "Div":"/", "Mod":"%",
"LShift":"<<", "RShift":">>", "BitOr":"|", "BitXor":"^", "BitAnd":"&",
"FloorDiv":"//", "Pow": "**"}
def _BinOp(self, t):
self.write("(")
self.dispatch(t.left)
self.write(" " + self.binop[t.op.__class__.__name__] + " ")
self.dispatch(t.right)
self.write(")")
cmpops = {"Eq":"==", "NotEq":"!=", "Lt":"<", "LtE":"<=", "Gt":">", "GtE":">=",
"Is":"is", "IsNot":"is not", "In":"in", "NotIn":"not in"}
def _Compare(self, t):
self.write("(")
self.dispatch(t.left)
for o, e in zip(t.ops, t.comparators):
self.write(" " + self.cmpops[o.__class__.__name__] + " ")
self.dispatch(e)
self.write(")")
boolops = {ast.And: 'and', ast.Or: 'or'}
def _BoolOp(self, t):
self.write("(")
s = " %s " % self.boolops[t.op.__class__]
interleave(lambda: self.write(s), self.dispatch, t.values)
self.write(")")
def _Attribute(self,t):
self.dispatch(t.value)
# Special case: 3.__abs__() is a syntax error, so if t.value
# is an integer literal then we need to either parenthesize
# it or add an extra space to get 3 .__abs__().
if isinstance(t.value, ast.Num) and isinstance(t.value.n, int):
self.write(" ")
self.write(".")
self.write(t.attr)
def _Call(self, t):
self.dispatch(t.func)
self.write("(")
comma = False
for e in t.args:
if comma: self.write(", ")
else: comma = True
self.dispatch(e)
for e in t.keywords:
if comma: self.write(", ")
else: comma = True
self.dispatch(e)
self.write(")")
def _Subscript(self, t):
self.dispatch(t.value)
self.write("[")
self.dispatch(t.slice)
self.write("]")
def _Starred(self, t):
self.write("*")
self.dispatch(t.value)
# slice
def _Ellipsis(self, t):
self.write("...")
def _Index(self, t):
self.dispatch(t.value)
def _Slice(self, t):
if t.lower:
self.dispatch(t.lower)
self.write(":")
if t.upper:
self.dispatch(t.upper)
if t.step:
self.write(":")
self.dispatch(t.step)
def _ExtSlice(self, t):
interleave(lambda: self.write(', '), self.dispatch, t.dims)
# argument
def _arg(self, t):
self.write(t.arg)
if t.annotation:
self.write(": ")
self.dispatch(t.annotation)
# others
def _arguments(self, t):
first = True
# normal arguments
defaults = [None] * (len(t.args) - len(t.defaults)) + t.defaults
for a, d in zip(t.args, defaults):
if first:first = False
else: self.write(", ")
self.dispatch(a)
if d:
self.write("=")
self.dispatch(d)
# varargs, or bare '*' if no varargs but keyword-only arguments present
if t.vararg or t.kwonlyargs:
if first:first = False
else: self.write(", ")
self.write("*")
if t.vararg:
self.write(t.vararg.arg)
if t.vararg.annotation:
self.write(": ")
self.dispatch(t.vararg.annotation)
# keyword-only arguments
if t.kwonlyargs:
for a, d in zip(t.kwonlyargs, t.kw_defaults):
if first:first = False
else: self.write(", ")
self.dispatch(a),
if d:
self.write("=")
self.dispatch(d)
# kwargs
if t.kwarg:
if first:first = False
else: self.write(", ")
self.write("**"+t.kwarg.arg)
if t.kwarg.annotation:
self.write(": ")
self.dispatch(t.kwarg.annotation)
def _keyword(self, t):
if t.arg is None:
self.write("**")
else:
self.write(t.arg)
self.write("=")
self.dispatch(t.value)
def _Lambda(self, t):
self.write("(")
self.write("lambda ")
self.dispatch(t.args)
self.write(": ")
self.dispatch(t.body)
self.write(")")
def _alias(self, t):
self.write(t.name)
if t.asname:
self.write(" as "+t.asname)
def _withitem(self, t):
self.dispatch(t.context_expr)
if t.optional_vars:
self.write(" as ")
self.dispatch(t.optional_vars)
def roundtrip(filename, outpath):
with open(filename, "rb") as pyfile:
encoding = tokenize.detect_encoding(pyfile.readline)[0]
with open(filename, "r", encoding=encoding) as pyfile:
source = pyfile.read()
tree = compile(source, filename, "exec", ast.PyCF_ONLY_AST)
with open(outpath, "w", encoding=encoding) as output:
Unparser(tree, output)
def strip_comments_and_docstrings(path):
tmp = path + ".tmp"
if path.endswith(".py"):
roundtrip(path, tmp)
else:
shutil.copy(path, tmp)
return tmp

2
python/ql/lib/BUILD.bazel
View File

@@ -1,4 +1,4 @@
load("@rules_pkg//:mappings.bzl", "pkg_files")
load("@rules_pkg//pkg:mappings.bzl", "pkg_files")
package(default_visibility = ["//python:__pkg__"])

15
python/ql/lib/semmle/python/dataflow/new/FlowSummary.qll
View File

@@ -31,8 +31,21 @@ abstract class SummarizedCallable extends LibraryCallable, Impl::Public::Summari
* DEPRECATED: Use `propagatesFlow` instead.
*/
deprecated predicate propagatesFlowExt(string input, string output, boolean preservesValue) {
this.propagatesFlow(input, output, preservesValue)
this.propagatesFlow(input, output, preservesValue, _)
}
override predicate propagatesFlow(
string input, string output, boolean preservesValue, string model
) {
this.propagatesFlow(input, output, preservesValue) and model = this
}
/**
* Holds if data may flow from `input` to `output` through this callable.
*
* `preservesValue` indicates whether this is a value-preserving step or a taint-step.
*/
predicate propagatesFlow(string input, string output, boolean preservesValue) { none() }
}
deprecated class RequiredSummaryComponentStack = Impl::Private::RequiredSummaryComponentStack;

5
python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl1.qll
View File

@@ -263,9 +263,10 @@ deprecated private module Config implements FullStateConfigSig {
predicate isBarrierOut(Node node, FlowState state) { none() }
predicate isAdditionalFlowStep(Node node1, Node node2) {
predicate isAdditionalFlowStep(Node node1, Node node2, string model) {
singleConfiguration() and
any(Configuration config).isAdditionalFlowStep(node1, node2)
any(Configuration config).isAdditionalFlowStep(node1, node2) and
model = ""
}
predicate isAdditionalFlowStep(Node node1, FlowState state1, Node node2, FlowState state2) {

5
python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl2.qll
View File

@@ -263,9 +263,10 @@ deprecated private module Config implements FullStateConfigSig {
predicate isBarrierOut(Node node, FlowState state) { none() }
predicate isAdditionalFlowStep(Node node1, Node node2) {
predicate isAdditionalFlowStep(Node node1, Node node2, string model) {
singleConfiguration() and
any(Configuration config).isAdditionalFlowStep(node1, node2)
any(Configuration config).isAdditionalFlowStep(node1, node2) and
model = ""
}
predicate isAdditionalFlowStep(Node node1, FlowState state1, Node node2, FlowState state2) {

5
python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl3.qll
View File

@@ -263,9 +263,10 @@ deprecated private module Config implements FullStateConfigSig {
predicate isBarrierOut(Node node, FlowState state) { none() }
predicate isAdditionalFlowStep(Node node1, Node node2) {
predicate isAdditionalFlowStep(Node node1, Node node2, string model) {
singleConfiguration() and
any(Configuration config).isAdditionalFlowStep(node1, node2)
any(Configuration config).isAdditionalFlowStep(node1, node2) and
model = ""
}
predicate isAdditionalFlowStep(Node node1, FlowState state1, Node node2, FlowState state2) {

5
python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl4.qll
View File

@@ -263,9 +263,10 @@ deprecated private module Config implements FullStateConfigSig {
predicate isBarrierOut(Node node, FlowState state) { none() }
predicate isAdditionalFlowStep(Node node1, Node node2) {
predicate isAdditionalFlowStep(Node node1, Node node2, string model) {
singleConfiguration() and
any(Configuration config).isAdditionalFlowStep(node1, node2)
any(Configuration config).isAdditionalFlowStep(node1, node2) and
model = ""
}
predicate isAdditionalFlowStep(Node node1, FlowState state1, Node node2, FlowState state2) {

21
python/ql/lib/semmle/python/dataflow/new/internal/DataFlowPrivate.qll
View File

@@ -3,6 +3,7 @@ private import DataFlowPublic
private import semmle.python.essa.SsaCompute
private import semmle.python.dataflow.new.internal.ImportResolution
private import FlowSummaryImpl as FlowSummaryImpl
private import semmle.python.frameworks.data.ModelsAsData
// Since we allow extra data-flow steps from modeled frameworks, we import these
// up-front, to ensure these are included. This provides a more seamless experience from
// a user point of view, since they don't need to know they need to import a specific
@@ -471,12 +472,12 @@ import StepRelationTransformations
*
* It includes flow steps from flow summaries.
*/
predicate simpleLocalFlowStep(Node nodeFrom, Node nodeTo) {
simpleLocalFlowStepForTypetracking(nodeFrom, nodeTo)
predicate simpleLocalFlowStep(Node nodeFrom, Node nodeTo, string model) {
simpleLocalFlowStepForTypetracking(nodeFrom, nodeTo) and model = ""
or
summaryLocalStep(nodeFrom, nodeTo)
summaryLocalStep(nodeFrom, nodeTo, model)
or
variableCaptureLocalFlowStep(nodeFrom, nodeTo)
variableCaptureLocalFlowStep(nodeFrom, nodeTo) and model = ""
}
/**
@@ -490,9 +491,9 @@ predicate simpleLocalFlowStepForTypetracking(Node nodeFrom, Node nodeTo) {
LocalFlow::localFlowStep(nodeFrom, nodeTo)
}
private predicate summaryLocalStep(Node nodeFrom, Node nodeTo) {
private predicate summaryLocalStep(Node nodeFrom, Node nodeTo, string model) {
FlowSummaryImpl::Private::Steps::summaryLocalStep(nodeFrom.(FlowSummaryNode).getSummaryNode(),
nodeTo.(FlowSummaryNode).getSummaryNode(), true)
nodeTo.(FlowSummaryNode).getSummaryNode(), true, model)
}
predicate variableCaptureLocalFlowStep(Node nodeFrom, Node nodeTo) {
@@ -1078,6 +1079,14 @@ predicate lambdaCall(DataFlowCall call, LambdaCallKind kind, Node receiver) {
/** Extra data-flow steps needed for lambda flow analysis. */
predicate additionalLambdaFlowStep(Node nodeFrom, Node nodeTo, boolean preservesValue) { none() }
predicate knownSourceModel(Node source, string model) {
source = ModelOutput::getASourceNode(_, model).asSource()
}
predicate knownSinkModel(Node sink, string model) {
sink = ModelOutput::getASinkNode(_, model).asSink()
}
/**
* Holds if flow is allowed to pass from parameter `p` and back to itself as a
* side-effect, resulting in a summary from `p` to itself.

4
python/ql/lib/semmle/python/dataflow/new/internal/DataFlowPublic.qll
View File

@@ -642,7 +642,9 @@ newtype TContent =
or
//
// 2) summaries in data-extension files
exists(string input, string output | ModelOutput::relevantSummaryModel(_, _, input, output, _) |
exists(string input, string output |
ModelOutput::relevantSummaryModel(_, _, input, output, _, _)
|
attr = [input, output].regexpFind("(?<=(^|\\.)Attribute\\[)[^\\]]+(?=\\])", _, _).trim()
)
} or

2
python/ql/lib/semmle/python/dataflow/new/internal/DataFlowUtil.qll
View File

@@ -12,7 +12,7 @@ private import FlowSummaryImpl as FlowSummaryImpl
* (intra-procedural) step.
*/
predicate localFlowStep(Node nodeFrom, Node nodeTo) {
simpleLocalFlowStep(nodeFrom, nodeTo)
simpleLocalFlowStep(nodeFrom, nodeTo, _)
or
// Simple flow through library code is included in the exposed local
// step relation, even though flow is technically inter-procedural.

2
python/ql/lib/semmle/python/dataflow/new/internal/LocalSources.qll
View File

@@ -242,7 +242,7 @@ private module Cached {
*/
pragma[nomagic]
private predicate localSourceFlowStep(Node nodeFrom, Node nodeTo) {
simpleLocalFlowStep(nodeFrom, nodeTo) and
simpleLocalFlowStep(nodeFrom, nodeTo, _) and
not nodeTo = any(ModuleVariableNode v).getARead()
}

53
python/ql/lib/semmle/python/dataflow/new/internal/TaintTrackingPrivate.qll
View File

@@ -24,10 +24,11 @@ private module Cached {
* global taint flow configurations.
*/
cached
predicate defaultAdditionalTaintStep(DataFlow::Node nodeFrom, DataFlow::Node nodeTo) {
localAdditionalTaintStep(nodeFrom, nodeTo)
predicate defaultAdditionalTaintStep(DataFlow::Node nodeFrom, DataFlow::Node nodeTo, string model) {
localAdditionalTaintStep(nodeFrom, nodeTo, model)
or
any(AdditionalTaintStep a).step(nodeFrom, nodeTo)
any(AdditionalTaintStep a).step(nodeFrom, nodeTo) and
model = "AdditionalTaintStep"
}
/**
@@ -36,30 +37,34 @@ private module Cached {
* different objects.
*/
cached
predicate localAdditionalTaintStep(DataFlow::Node nodeFrom, DataFlow::Node nodeTo) {
concatStep(nodeFrom, nodeTo)
or
subscriptStep(nodeFrom, nodeTo)
or
stringManipulation(nodeFrom, nodeTo)
or
containerStep(nodeFrom, nodeTo)
or
copyStep(nodeFrom, nodeTo)
or
DataFlowPrivate::forReadStep(nodeFrom, _, nodeTo)
or
DataFlowPrivate::iterableUnpackingReadStep(nodeFrom, _, nodeTo)
or
DataFlowPrivate::iterableUnpackingStoreStep(nodeFrom, _, nodeTo)
or
awaitStep(nodeFrom, nodeTo)
or
asyncWithStep(nodeFrom, nodeTo)
predicate localAdditionalTaintStep(DataFlow::Node nodeFrom, DataFlow::Node nodeTo, string model) {
(
concatStep(nodeFrom, nodeTo)
or
subscriptStep(nodeFrom, nodeTo)
or
stringManipulation(nodeFrom, nodeTo)
or
containerStep(nodeFrom, nodeTo)
or
copyStep(nodeFrom, nodeTo)
or
DataFlowPrivate::forReadStep(nodeFrom, _, nodeTo)
or
DataFlowPrivate::iterableUnpackingReadStep(nodeFrom, _, nodeTo)
or
DataFlowPrivate::iterableUnpackingStoreStep(nodeFrom, _, nodeTo)
or
awaitStep(nodeFrom, nodeTo)
or
asyncWithStep(nodeFrom, nodeTo)
) and
model = ""
or
FlowSummaryImpl::Private::Steps::summaryLocalStep(nodeFrom
.(DataFlowPrivate::FlowSummaryNode)
.getSummaryNode(), nodeTo.(DataFlowPrivate::FlowSummaryNode).getSummaryNode(), false)
.getSummaryNode(), nodeTo.(DataFlowPrivate::FlowSummaryNode).getSummaryNode(), false,
model)
}
}

2
python/ql/lib/semmle/python/dataflow/new/internal/TaintTrackingPublic.qll
View File

@@ -34,7 +34,7 @@ predicate localTaintStep(DataFlow::Node nodeFrom, DataFlow::Node nodeTo) {
// Ordinary data flow
DataFlow::localFlowStep(nodeFrom, nodeTo)
or
localAdditionalTaintStep(nodeFrom, nodeTo)
localAdditionalTaintStep(nodeFrom, nodeTo, _)
}
/**

10
python/ql/lib/semmle/python/dataflow/new/internal/TypeTrackingImpl.qll
View File

@@ -24,7 +24,15 @@ private module SummaryTypeTrackerInput implements SummaryTypeTracker::Input {
ContentFilter getFilterFromWithContentStep(Content content) { none() }
// Callables
class SummarizedCallable = FlowSummaryImpl::Private::SummarizedCallableImpl;
class SummarizedCallable instanceof FlowSummaryImpl::Private::SummarizedCallableImpl {
string toString() { result = super.toString() }
predicate propagatesFlow(
SummaryComponentStack input, SummaryComponentStack output, boolean preservesValue
) {
super.propagatesFlow(input, output, preservesValue, _)
}
}
// Summaries and their stacks
class SummaryComponent = FlowSummaryImpl::Private::SummaryComponent;

2
python/ql/lib/semmle/python/dataflow/new/internal/tainttracking1/TaintTrackingImpl.qll
View File

@@ -127,7 +127,7 @@ abstract deprecated class Configuration extends DataFlow::Configuration {
final override predicate isAdditionalFlowStep(DataFlow::Node node1, DataFlow::Node node2) {
this.isAdditionalTaintStep(node1, node2) or
defaultAdditionalTaintStep(node1, node2)
defaultAdditionalTaintStep(node1, node2, _)
}
/**

2
python/ql/lib/semmle/python/dataflow/new/internal/tainttracking2/TaintTrackingImpl.qll
View File

@@ -127,7 +127,7 @@ abstract deprecated class Configuration extends DataFlow::Configuration {
final override predicate isAdditionalFlowStep(DataFlow::Node node1, DataFlow::Node node2) {
this.isAdditionalTaintStep(node1, node2) or
defaultAdditionalTaintStep(node1, node2)
defaultAdditionalTaintStep(node1, node2, _)
}
/**

2
python/ql/lib/semmle/python/dataflow/new/internal/tainttracking3/TaintTrackingImpl.qll
View File

@@ -127,7 +127,7 @@ abstract deprecated class Configuration extends DataFlow::Configuration {
final override predicate isAdditionalFlowStep(DataFlow::Node node1, DataFlow::Node node2) {
this.isAdditionalTaintStep(node1, node2) or
defaultAdditionalTaintStep(node1, node2)
defaultAdditionalTaintStep(node1, node2, _)
}
/**

2
python/ql/lib/semmle/python/dataflow/new/internal/tainttracking4/TaintTrackingImpl.qll
View File

@@ -127,7 +127,7 @@ abstract deprecated class Configuration extends DataFlow::Configuration {
final override predicate isAdditionalFlowStep(DataFlow::Node node1, DataFlow::Node node2) {
this.isAdditionalTaintStep(node1, node2) or
defaultAdditionalTaintStep(node1, node2)
defaultAdditionalTaintStep(node1, node2, _)
}
/**

29
python/ql/lib/semmle/python/frameworks/Asyncpg.model.yml Normal file
View File

@@ -0,0 +1,29 @@
extensions:
- addsTo:
pack: codeql/python-all
extensible: sinkModel
data:
# `Connection`s and `ConnectionPool`s provide some methods that execute SQL.
- ['asyncpg.~Connection', 'Member[copy_from_query,execute,fetch,fetchrow,fetchval].Argument[0,query:]', 'sql-injection']
- ['asyncpg.~Connection', 'Member[executemany].Argument[0,command:]', 'sql-injection']
# A model of `Connection` and `ConnectionPool`, which provide some methods that access the file system.
- ['asyncpg.~Connection', 'Member[copy_from_query,copy_from_table].Argument[output:]', 'path-injection']
- ['asyncpg.~Connection', 'Member[copy_to_table].Argument[source:]', 'path-injection']
# the `PreparedStatement` class in `asyncpg`.
- ['asyncpg.Connection', 'Member[prepare].Argument[0,query:]', 'sql-injection']
- addsTo:
pack: codeql/python-all
extensible: typeModel
data:
# a `ConnectionPool` that is created when the result of `asyncpg.create_pool()` is awaited.
- ['asyncpg.Connection', 'asyncpg.ConnectionPool', 'Member[acquire].ReturnValue.Awaited']
# a `Connection` that is created when
# * - the result of `asyncpg.connect()` is awaited.
# * - the result of calling `acquire` on a `ConnectionPool` is awaited.
- ['asyncpg.Connection', 'asyncpg', 'Member[connect].ReturnValue.Awaited']
- ['asyncpg.Connection', 'asyncpg', 'Member[connection].Member[connect].ReturnValue.Awaited']
- ['asyncpg.ConnectionPool', 'asyncpg', 'Member[create_pool].ReturnValue.Awaited']
# Creating an internal `~Connection` type that contains both `Connection` and `ConnectionPool`.
- ['asyncpg.~Connection', 'asyncpg.Connection', '']
- ['asyncpg.~Connection', 'asyncpg.ConnectionPool', '']

37
python/ql/lib/semmle/python/frameworks/Asyncpg.qll
View File

@@ -11,43 +11,6 @@ private import semmle.python.frameworks.data.ModelsAsData
/** Provides models for the `asyncpg` PyPI package. */
private module Asyncpg {
class AsyncpgModel extends ModelInput::TypeModelCsv {
override predicate row(string row) {
// type1;type2;path
row =
[
// a `ConnectionPool` that is created when the result of `asyncpg.create_pool()` is awaited.
"asyncpg.ConnectionPool;asyncpg;Member[create_pool].ReturnValue.Awaited",
// a `Connection` that is created when
// * - the result of `asyncpg.connect()` is awaited.
// * - the result of calling `acquire` on a `ConnectionPool` is awaited.
"asyncpg.Connection;asyncpg;Member[connect].ReturnValue.Awaited",
"asyncpg.Connection;asyncpg;Member[connection].Member[connect].ReturnValue.Awaited",
"asyncpg.Connection;asyncpg.ConnectionPool;Member[acquire].ReturnValue.Awaited",
// Creating an internal `~Connection` type that contains both `Connection` and `ConnectionPool`.
"asyncpg.~Connection;asyncpg.Connection;", //
"asyncpg.~Connection;asyncpg.ConnectionPool;"
]
}
}
class AsyncpgSink extends ModelInput::SinkModelCsv {
// type;path;kind
override predicate row(string row) {
row =
[
// `Connection`s and `ConnectionPool`s provide some methods that execute SQL.
"asyncpg.~Connection;Member[copy_from_query,execute,fetch,fetchrow,fetchval].Argument[0,query:];sql-injection",
"asyncpg.~Connection;Member[executemany].Argument[0,command:];sql-injection",
// A model of `Connection` and `ConnectionPool`, which provide some methods that access the file system.
"asyncpg.~Connection;Member[copy_from_query,copy_from_table].Argument[output:];path-injection",
"asyncpg.~Connection;Member[copy_to_table].Argument[source:];path-injection",
// the `PreparedStatement` class in `asyncpg`.
"asyncpg.Connection;Member[prepare].Argument[0,query:];sql-injection",
]
}
}
/**
* Provides models of the `Cursor` class in `asyncpg`.
* `Cursor`s are created

10
python/ql/lib/semmle/python/frameworks/data/ModelsAsData.qll
View File

@@ -22,7 +22,7 @@ private import semmle.python.dataflow.new.FlowSummary
/**
* A remote flow source originating from a CSV source row.
*/
private class RemoteFlowSourceFromCsv extends RemoteFlowSource {
private class RemoteFlowSourceFromCsv extends RemoteFlowSource::Range {
RemoteFlowSourceFromCsv() { this = ModelOutput::getASourceNode("remote").asSource() }
override string getSourceType() { result = "Remote flow (from model)" }
@@ -33,7 +33,7 @@ private class SummarizedCallableFromModel extends SummarizedCallable {
string path;
SummarizedCallableFromModel() {
ModelOutput::relevantSummaryModel(type, path, _, _, _) and
ModelOutput::relevantSummaryModel(type, path, _, _, _, _) and
this = type + ";" + path
}
@@ -46,8 +46,10 @@ private class SummarizedCallableFromModel extends SummarizedCallable {
)
}
override predicate propagatesFlow(string input, string output, boolean preservesValue) {
exists(string kind | ModelOutput::relevantSummaryModel(type, path, input, output, kind) |
override predicate propagatesFlow(
string input, string output, boolean preservesValue, string model
) {
exists(string kind | ModelOutput::relevantSummaryModel(type, path, input, output, kind, model) |
kind = "value" and
preservesValue = true
or

264
python/ql/lib/semmle/python/frameworks/data/internal/ApiGraphModels.qll
View File

@@ -1,17 +1,17 @@
/**
* INTERNAL use only. This is an experimental API subject to change without notice.
*
* Provides classes and predicates for dealing with flow models specified in CSV format.
* Provides classes and predicates for dealing with flow models specified in extensible predicates.
*
* The CSV specification has the following columns:
* The extensible predicates have the following columns:
* - Sources:
* `type; path; kind`
* `type, path, kind`
* - Sinks:
* `type; path; kind`
* `type, path, kind`
* - Summaries:
* `type; path; input; output; kind`
* `type, path, input, output, kind`
* - Types:
* `type1; type2; path`
* `type1, type2, path`
*
* The interpretation of a row is similar to API-graphs with a left-to-right
* reading.
@@ -76,11 +76,13 @@ private import codeql.dataflow.internal.AccessPathSyntax
/** Module containing hooks for providing input data to be interpreted as a model. */
module ModelInput {
/**
* DEPRECATED: Use the extensible predicate `sourceModel` instead.
*
* A unit class for adding additional source model rows.
*
* Extend this class to add additional source definitions.
*/
class SourceModelCsv extends Unit {
deprecated class SourceModelCsv extends Unit {
/**
* Holds if `row` specifies a source definition.
*
@@ -93,15 +95,17 @@ module ModelInput {
*
* The kind `remote` represents a general remote flow source.
*/
abstract predicate row(string row);
abstract deprecated predicate row(string row);
}
/**
* A unit class for adding additional sink model rows.
*
* Extend this class to add additional sink definitions.
*
* DEPRECATED: Use the extensible predicate `sinkModel` instead.
*/
class SinkModelCsv extends Unit {
deprecated class SinkModelCsv extends Unit {
/**
* Holds if `row` specifies a sink definition.
*
@@ -112,15 +116,17 @@ module ModelInput {
* indicates that the value at `(type, path)` should be seen as a sink
* of the given `kind`.
*/
abstract predicate row(string row);
abstract deprecated predicate row(string row);
}
/**
* A unit class for adding additional summary model rows.
*
* Extend this class to add additional flow summary definitions.
*
* DEPRECATED: Use the extensible predicate `summaryModel` instead.
*/
class SummaryModelCsv extends Unit {
deprecated class SummaryModelCsv extends Unit {
/**
* Holds if `row` specifies a summary definition.
*
@@ -134,15 +140,18 @@ module ModelInput {
* `kind` should be either `value` or `taint`, for value-preserving or taint-preserving steps,
* respectively.
*/
abstract predicate row(string row);
abstract deprecated predicate row(string row);
}
/**
* A unit class for adding additional type model rows.
*
* Extend this class to add additional type definitions.
*
* DEPRECATED: Use the extensible predicate `typeModel` or the class
* `TypeModel` instead.
*/
class TypeModelCsv extends Unit {
deprecated class TypeModelCsv extends Unit {
/**
* Holds if `row` specifies a type definition.
*
@@ -152,7 +161,7 @@ module ModelInput {
* ```
* indicates that `(type2, path)` should be seen as an instance of `type1`.
*/
abstract predicate row(string row);
abstract deprecated predicate row(string row);
}
/**
@@ -186,8 +195,10 @@ module ModelInput {
/**
* A unit class for adding additional type variable model rows.
*
* DEPRECATED: Use the extensible predicate `typeVariableModel` instead.
*/
class TypeVariableModelCsv extends Unit {
deprecated class TypeVariableModelCsv extends Unit {
/**
* Holds if `row` specifies a path through a type variable.
*
@@ -197,7 +208,7 @@ module ModelInput {
* ```
* means `path` can be substituted for a token `TypeVar[name]`.
*/
abstract predicate row(string row);
abstract deprecated predicate row(string row);
}
}
@@ -216,57 +227,121 @@ abstract class TestAllModels extends Unit { }
* does not preserve empty trailing substrings.
*/
bindingset[result]
private string inversePad(string s) { s = result + ";dummy" }
deprecated private string inversePad(string s) { s = result + ";dummy" }
private predicate sourceModel(string row) { any(SourceModelCsv s).row(inversePad(row)) }
deprecated private predicate sourceModel(string row) { any(SourceModelCsv s).row(inversePad(row)) }
private predicate sinkModel(string row) { any(SinkModelCsv s).row(inversePad(row)) }
deprecated private predicate sinkModel(string row) { any(SinkModelCsv s).row(inversePad(row)) }
private predicate summaryModel(string row) { any(SummaryModelCsv s).row(inversePad(row)) }
deprecated private predicate summaryModel(string row) {
any(SummaryModelCsv s).row(inversePad(row))
}
private predicate typeModel(string row) { any(TypeModelCsv s).row(inversePad(row)) }
deprecated private predicate typeModel(string row) { any(TypeModelCsv s).row(inversePad(row)) }
private predicate typeVariableModel(string row) { any(TypeVariableModelCsv s).row(inversePad(row)) }
deprecated private predicate typeVariableModel(string row) {
any(TypeVariableModelCsv s).row(inversePad(row))
}
private class DeprecationAdapter extends Unit {
abstract predicate sourceModel(string type, string path, string kind);
abstract predicate sinkModel(string type, string path, string kind);
abstract predicate summaryModel(string type, string path, string input, string output, string kind);
abstract predicate typeModel(string type1, string type2, string path);
abstract predicate typeVariableModel(string name, string path);
}
private class DeprecationAdapterImpl extends DeprecationAdapter {
deprecated override predicate sourceModel(string type, string path, string kind) {
exists(string row |
sourceModel(row) and
row.splitAt(";", 0) = type and
row.splitAt(";", 1) = path and
row.splitAt(";", 2) = kind
)
}
deprecated override predicate sinkModel(string type, string path, string kind) {
exists(string row |
sinkModel(row) and
row.splitAt(";", 0) = type and
row.splitAt(";", 1) = path and
row.splitAt(";", 2) = kind
)
}
deprecated override predicate summaryModel(
string type, string path, string input, string output, string kind
) {
exists(string row |
summaryModel(row) and
row.splitAt(";", 0) = type and
row.splitAt(";", 1) = path and
row.splitAt(";", 2) = input and
row.splitAt(";", 3) = output and
row.splitAt(";", 4) = kind
)
}
deprecated override predicate typeModel(string type1, string type2, string path) {
exists(string row |
typeModel(row) and
row.splitAt(";", 0) = type1 and
row.splitAt(";", 1) = type2 and
row.splitAt(";", 2) = path
)
}
deprecated override predicate typeVariableModel(string name, string path) {
exists(string row |
typeVariableModel(row) and
row.splitAt(";", 0) = name and
row.splitAt(";", 1) = path
)
}
}
/** Holds if a source model exists for the given parameters. */
predicate sourceModel(string type, string path, string kind) {
exists(string row |
sourceModel(row) and
row.splitAt(";", 0) = type and
row.splitAt(";", 1) = path and
row.splitAt(";", 2) = kind
)
predicate sourceModel(string type, string path, string kind, string model) {
any(DeprecationAdapter a).sourceModel(type, path, kind) and
model = "SourceModelCsv"
or
Extensions::sourceModel(type, path, kind)
exists(QlBuiltins::ExtensionId madId |
Extensions::sourceModel(type, path, kind, madId) and
model = "MaD:" + madId.toString()
)
}
/** Holds if a sink model exists for the given parameters. */
private predicate sinkModel(string type, string path, string kind) {
exists(string row |
sinkModel(row) and
row.splitAt(";", 0) = type and
row.splitAt(";", 1) = path and
row.splitAt(";", 2) = kind
)
private predicate sinkModel(string type, string path, string kind, string model) {
any(DeprecationAdapter a).sinkModel(type, path, kind) and
model = "SinkModelCsv"
or
Extensions::sinkModel(type, path, kind)
exists(QlBuiltins::ExtensionId madId |
Extensions::sinkModel(type, path, kind, madId) and
model = "MaD:" + madId.toString()
)
}
/** Holds if a summary model `row` exists for the given parameters. */
private predicate summaryModel(string type, string path, string input, string output, string kind) {
exists(string row |
summaryModel(row) and
row.splitAt(";", 0) = type and
row.splitAt(";", 1) = path and
row.splitAt(";", 2) = input and
row.splitAt(";", 3) = output and
row.splitAt(";", 4) = kind
)
private predicate summaryModel(
string type, string path, string input, string output, string kind, string model
) {
any(DeprecationAdapter a).summaryModel(type, path, input, output, kind) and
model = "SummaryModelCsv"
or
Extensions::summaryModel(type, path, input, output, kind)
exists(QlBuiltins::ExtensionId madId |
Extensions::summaryModel(type, path, input, output, kind, madId) and
model = "MaD:" + madId.toString()
)
}
/** Holds if a type model exists for the given parameters. */
<<<<<<< HEAD
predicate typeModel(string type1, string type2, string path) {
exists(string row |
typeModel(row) and
@@ -274,29 +349,29 @@ predicate typeModel(string type1, string type2, string path) {
row.splitAt(";", 1) = type2 and
row.splitAt(";", 2) = path
)
=======
private predicate typeModel(string type1, string type2, string path) {
any(DeprecationAdapter a).typeModel(type1, type2, path)
>>>>>>> main
or
Extensions::typeModel(type1, type2, path)
}
/** Holds if a type variable model exists for the given parameters. */
private predicate typeVariableModel(string name, string path) {
exists(string row |
typeVariableModel(row) and
row.splitAt(";", 0) = name and
row.splitAt(";", 1) = path
)
any(DeprecationAdapter a).typeVariableModel(name, path)
or
Extensions::typeVariableModel(name, path)
}
/**
* Holds if CSV rows involving `type` might be relevant for the analysis of this database.
* Holds if rows involving `type` might be relevant for the analysis of this database.
*/
predicate isRelevantType(string type) {
(
sourceModel(type, _, _) or
sinkModel(type, _, _) or
summaryModel(type, _, _, _, _) or
sourceModel(type, _, _, _) or
sinkModel(type, _, _, _) or
summaryModel(type, _, _, _, _, _) or
typeModel(_, type, _)
) and
(
@@ -313,26 +388,26 @@ predicate isRelevantType(string type) {
}
/**
* Holds if `type,path` is used in some CSV row.
* Holds if `type,path` is used in some row.
*/
pragma[nomagic]
predicate isRelevantFullPath(string type, string path) {
isRelevantType(type) and
(
sourceModel(type, path, _) or
sinkModel(type, path, _) or
summaryModel(type, path, _, _, _) or
sourceModel(type, path, _, _) or
sinkModel(type, path, _, _) or
summaryModel(type, path, _, _, _, _) or
typeModel(_, type, path)
)
}
/** A string from a CSV row that should be parsed as an access path. */
/** A string from a row that should be parsed as an access path. */
private predicate accessPathRange(string s) {
isRelevantFullPath(_, s)
or
exists(string type | isRelevantType(type) |
summaryModel(type, _, s, _, _) or
summaryModel(type, _, _, s, _)
summaryModel(type, _, s, _, _, _) or
summaryModel(type, _, _, s, _, _)
)
or
typeVariableModel(_, s)
@@ -543,7 +618,7 @@ private API::Node getNodeFromPath(string type, AccessPath path) {
pragma[nomagic]
private predicate typeStepModel(string type, AccessPath basePath, AccessPath output) {
summaryModel(type, basePath, "", output, "type")
summaryModel(type, basePath, "", output, "type", _)
}
pragma[nomagic]
@@ -618,36 +693,36 @@ module ModelOutput {
cached
private module Cached {
/**
* Holds if a CSV source model contributed `source` with the given `kind`.
* Holds if a source model contributed `source` with the given `kind`.
*/
cached
API::Node getASourceNode(string kind) {
API::Node getASourceNode(string kind, string model) {
exists(string type, string path |
sourceModel(type, path, kind) and
sourceModel(type, path, kind, model) and
result = getNodeFromPath(type, path)
)
}
/**
* Holds if a CSV sink model contributed `sink` with the given `kind`.
* Holds if a sink model contributed `sink` with the given `kind`.
*/
cached
API::Node getASinkNode(string kind) {
API::Node getASinkNode(string kind, string model) {
exists(string type, string path |
sinkModel(type, path, kind) and
sinkModel(type, path, kind, model) and
result = getNodeFromPath(type, path)
)
}
/**
* Holds if a relevant CSV summary exists for these parameters.
* Holds if a relevant summary exists for these parameters.
*/
cached
predicate relevantSummaryModel(
string type, string path, string input, string output, string kind
string type, string path, string input, string output, string kind, string model
) {
isRelevantType(type) and
summaryModel(type, path, input, output, kind)
summaryModel(type, path, input, output, kind, model)
}
/**
@@ -655,7 +730,7 @@ module ModelOutput {
*/
cached
predicate resolvedSummaryBase(string type, string path, Specific::InvokeNode baseNode) {
summaryModel(type, path, _, _, _) and
summaryModel(type, path, _, _, _, _) and
baseNode = getInvocationFromPath(type, path)
}
@@ -664,13 +739,13 @@ module ModelOutput {
*/
cached
predicate resolvedSummaryRefBase(string type, string path, API::Node baseNode) {
summaryModel(type, path, _, _, _) and
summaryModel(type, path, _, _, _, _) and
baseNode = getNodeFromPath(type, path)
}
/**
* Holds if `node` is seen as an instance of `type` due to a type definition
* contributed by a CSV model.
* contributed by a model.
*/
cached
API::Node getATypeNode(string type) { result = getNodeFromType(type) }
@@ -680,12 +755,22 @@ module ModelOutput {
import Specific::ModelOutputSpecific
private import codeql.mad.ModelValidation as SharedModelVal
/**
* Holds if a CSV source model contributed `source` with the given `kind`.
*/
API::Node getASourceNode(string kind) { result = getASourceNode(kind, _) }
/**
* Holds if a CSV sink model contributed `sink` with the given `kind`.
*/
API::Node getASinkNode(string kind) { result = getASinkNode(kind, _) }
private module KindValConfig implements SharedModelVal::KindValidationConfigSig {
predicate summaryKind(string kind) { summaryModel(_, _, _, _, kind) }
predicate summaryKind(string kind) { summaryModel(_, _, _, _, kind, _) }
predicate sinkKind(string kind) { sinkModel(_, _, kind) }
predicate sinkKind(string kind) { sinkModel(_, _, kind, _) }
predicate sourceKind(string kind) { sourceModel(_, _, kind) }
predicate sourceKind(string kind) { sourceModel(_, _, kind, _) }
}
private module KindVal = SharedModelVal::KindValidation<KindValConfig>;
@@ -694,25 +779,6 @@ module ModelOutput {
* Gets an error message relating to an invalid CSV row in a model.
*/
string getAWarning() {
// Check number of columns
exists(string row, string kind, int expectedArity, int actualArity |
any(SourceModelCsv csv).row(row) and kind = "source" and expectedArity = 3
or
any(SinkModelCsv csv).row(row) and kind = "sink" and expectedArity = 3
or
any(SummaryModelCsv csv).row(row) and kind = "summary" and expectedArity = 5
or
any(TypeModelCsv csv).row(row) and kind = "type" and expectedArity = 3
or
any(TypeVariableModelCsv csv).row(row) and kind = "type-variable" and expectedArity = 2
|
actualArity = count(row.indexOf(";")) + 1 and
actualArity != expectedArity and
result =
"CSV " + kind + " row should have " + expectedArity + " columns but has " + actualArity +
": " + row
)
or
// Check names and arguments of access path tokens
exists(AccessPath path, AccessPathToken token |
(isRelevantFullPath(_, path) or typeVariableModel(_, path)) and

10
python/ql/lib/semmle/python/frameworks/data/internal/ApiGraphModelsExtensions.qll
View File

@@ -8,13 +8,15 @@
*
* The kind `remote` represents a general remote flow source.
*/
extensible predicate sourceModel(string type, string path, string kind);
extensible predicate sourceModel(
string type, string path, string kind, QlBuiltins::ExtensionId madId
);
/**
* Holds if the value at `(type, path)` should be seen as a sink
* of the given `kind`.
*/
extensible predicate sinkModel(string type, string path, string kind);
extensible predicate sinkModel(string type, string path, string kind, QlBuiltins::ExtensionId madId);
/**
* Holds if in calls to `(type, path)`, the value referred to by `input`
@@ -23,7 +25,9 @@ extensible predicate sinkModel(string type, string path, string kind);
* `kind` should be either `value` or `taint`, for value-preserving or taint-preserving steps,
* respectively.
*/
extensible predicate summaryModel(string type, string path, string input, string output, string kind);
extensible predicate summaryModel(
string type, string path, string input, string output, string kind, QlBuiltins::ExtensionId madId
);
/**
* Holds if calls to `(type, path)` should be considered neutral. The meaning of this depends on the `kind`.

4
python/ql/lib/semmle/python/frameworks/data/internal/ApiGraphModelsSpecific.qll
View File

@@ -138,7 +138,7 @@ predicate invocationMatchesExtraCallSiteFilter(API::CallNode invoke, AccessPathT
pragma[nomagic]
private predicate relevantInputOutputPath(API::CallNode base, AccessPath inputOrOutput) {
exists(string type, string input, string output, string path |
ModelOutput::relevantSummaryModel(type, path, input, output, _) and
ModelOutput::relevantSummaryModel(type, path, input, output, _, _) and
ModelOutput::resolvedSummaryBase(type, path, base) and
inputOrOutput = [input, output]
)
@@ -170,7 +170,7 @@ private API::Node getNodeFromInputOutputPath(API::CallNode baseNode, AccessPath
*/
predicate summaryStep(API::Node pred, API::Node succ, string kind) {
exists(string type, string path, API::CallNode base, AccessPath input, AccessPath output |
ModelOutput::relevantSummaryModel(type, path, input, output, kind) and
ModelOutput::relevantSummaryModel(type, path, input, output, kind, _) and // TODO???
ModelOutput::resolvedSummaryBase(type, path, base) and
pred = getNodeFromInputOutputPath(base, input) and
succ = getNodeFromInputOutputPath(base, output)

4
python/ql/src/Security/CWE-020-ExternalAPIs/ExternalAPIs.qll
View File

@@ -157,12 +157,12 @@ class ExternalApiDataNode extends DataFlow::Node {
ExternalApiDataNode() {
exists(InterestingExternalApiCall call | this = call.getArgument(_)) and
// Not already modeled as a taint step
not TaintTrackingPrivate::defaultAdditionalTaintStep(this, _) and
not TaintTrackingPrivate::defaultAdditionalTaintStep(this, _, _) and
// for `list.append(x)`, we have a additional taint step from x -> [post] list.
// Since we have modeled this explicitly, I don't see any cases where we would want to report this.
not exists(DataFlow::PostUpdateNode post |
post.getPreUpdateNode() = this and
TaintTrackingPrivate::defaultAdditionalTaintStep(_, post)
TaintTrackingPrivate::defaultAdditionalTaintStep(_, post, _)
)
}
}

1
python/ql/test/2/extractor-tests/ellipsis/Test.expected Normal file
View File

@@ -0,0 +1 @@
| 2 |

3
python/ql/test/2/extractor-tests/ellipsis/Test.ql Normal file
View File

@@ -0,0 +1,3 @@
import python
select count(Ellipsis e)

6
python/ql/test/2/extractor-tests/ellipsis/test.py Normal file
View File

@@ -0,0 +1,6 @@
a = {}
a[...] = 1
b = a[
...]
print(b)

23
python/ql/test/2/extractor-tests/exec/successors.expected Normal file
View File

@@ -0,0 +1,23 @@
| test.py | 0 | Entry node for Module test | 1 | ControlFlowNode for FunctionExpr | normal |
| test.py | 1 | ControlFlowNode for FunctionExpr | 1 | ControlFlowNode for f | normal |
| test.py | 1 | ControlFlowNode for f | 0 | Exit node for Module test | normal |
| test.py | 1 | ControlFlowNode for x | 2 | ControlFlowNode for exec | normal |
| test.py | 1 | Entry node for Function f | 1 | ControlFlowNode for x | normal |
| test.py | 2 | ControlFlowNode for Str | 2 | ControlFlowNode for exec() | normal |
| test.py | 2 | ControlFlowNode for exec | 2 | ControlFlowNode for Str | normal |
| test.py | 2 | ControlFlowNode for exec() | 3 | ControlFlowNode for x | normal |
| test.py | 3 | ControlFlowNode for Return | 1 | Exit node for Function f | normal |
| test.py | 3 | ControlFlowNode for x | 3 | ControlFlowNode for Return | normal |
| unicode.py | 0 | Entry node for Module unicode | 1 | ControlFlowNode for ImportExpr | normal |
| unicode.py | 1 | ControlFlowNode for ImportExpr | 1 | ControlFlowNode for ImportMember | normal |
| unicode.py | 1 | ControlFlowNode for ImportMember | 1 | ControlFlowNode for unicode_literals | normal |
| unicode.py | 1 | ControlFlowNode for unicode_literals | 3 | ControlFlowNode for FunctionExpr | normal |
| unicode.py | 3 | ControlFlowNode for FunctionExpr | 3 | ControlFlowNode for f | normal |
| unicode.py | 3 | ControlFlowNode for f | 0 | Exit node for Module unicode | normal |
| unicode.py | 3 | ControlFlowNode for x | 4 | ControlFlowNode for exec | normal |
| unicode.py | 3 | Entry node for Function f | 3 | ControlFlowNode for x | normal |
| unicode.py | 4 | ControlFlowNode for Str | 4 | ControlFlowNode for exec() | normal |
| unicode.py | 4 | ControlFlowNode for exec | 4 | ControlFlowNode for Str | normal |
| unicode.py | 4 | ControlFlowNode for exec() | 5 | ControlFlowNode for x | normal |
| unicode.py | 5 | ControlFlowNode for Return | 3 | Exit node for Function f | normal |
| unicode.py | 5 | ControlFlowNode for x | 5 | ControlFlowNode for Return | normal |

14
python/ql/test/2/extractor-tests/exec/successors.ql Normal file
View File

@@ -0,0 +1,14 @@
import python
from ControlFlowNode p, ControlFlowNode s, string kind, string filename
where
p.getASuccessor() = s and
(
p.getAnExceptionalSuccessor() = s and kind = "exception"
or
not p.getAnExceptionalSuccessor() = s and kind = "normal"
) and
filename = p.getLocation().getFile().getShortName() and
not filename = "__future__.py"
select filename, p.getLocation().getStartLine(), p.toString(), s.getLocation().getStartLine(),
s.toString(), kind

3
python/ql/test/2/extractor-tests/exec/test.py Normal file
View File

@@ -0,0 +1,3 @@
def f(x):
exec("raise thing")
return x

5
python/ql/test/2/extractor-tests/exec/unicode.py Normal file
View File

@@ -0,0 +1,5 @@
from __future__ import unicode_literals
def f(x):
exec("raise thing")
return x

0
python/ql/test/query-tests/Security/lib/Crypto/PublicKey/__init__.py → python/ql/test/2/extractor-tests/hidden/.hidden/inner/test.py
View File

0
python/ql/test/query-tests/Security/lib/Crypto/__init__.py → python/ql/test/2/extractor-tests/hidden/.hidden/module.py
View File

0
python/ql/test/query-tests/Security/lib/cryptography/__init__.py → python/ql/test/2/extractor-tests/hidden/folder/module.py
View File

2
python/ql/test/2/extractor-tests/hidden/options Normal file
View File

@@ -0,0 +1,2 @@
semmle-extractor-options: -R . -p . --filter exclude:**/src_archive/**

0
python/ql/test/query-tests/Security/lib/cryptography/hazmat/__init__.py → python/ql/test/2/extractor-tests/hidden/package/__init__.py
View File

0
python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/__init__.py → python/ql/test/2/extractor-tests/hidden/package/module.py
View File

4
python/ql/test/2/extractor-tests/hidden/test.expected Normal file
View File

@@ -0,0 +1,4 @@
| folder/module.py |
| package |
| package/__init__.py |
| package/module.py |

5
python/ql/test/2/extractor-tests/hidden/test.ql Normal file
View File

@@ -0,0 +1,5 @@
import python
from Container f
where exists(Module m | m.getPath() = f)
select f.toString()

1
python/ql/test/2/extractor-tests/import_depth/options Normal file
View File

@@ -0,0 +1 @@
semmle-extractor-options: --lang=2 --max-import-depth=1

0
python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/asymmetric/__init__.py → python/ql/test/2/extractor-tests/import_depth/package/__init__.py
View File

0
python/ql/test/query-tests/Security/lib/django/__init__.py → python/ql/test/2/extractor-tests/import_depth/package/x.py
View File

4
python/ql/test/2/extractor-tests/import_depth/test.expected Normal file
View File

@@ -0,0 +1,4 @@
| Module package |
| Module package.__init__ |
| Module sys |
| Module test |

1
python/ql/test/2/extractor-tests/import_depth/test.py Normal file
View File

@@ -0,0 +1 @@
import package

6
python/ql/test/2/extractor-tests/import_depth/test.ql Normal file
View File

@@ -0,0 +1,6 @@
import python
from ModuleObject m
/* Exclude the builtins module as it has a different name under 2 and 3. */
where not m = theBuiltinModuleObject()
select m.toString()

6
python/ql/test/2/extractor-tests/multibyte/Test.expected Normal file
View File

@@ -0,0 +1,6 @@
| test.py:3:1:3:11 | Str | \u0111\u0142e\u00b6\u014b\u00b6\u0142\u014b |
| test.py:3:15:3:19 | Str | hi |
| test.py:3:23:3:27 | Str | \n |
| test.py:4:1:4:11 | Str | \u0111\u0142e\u00b6\u014b\u00b6\u0142\u014b |
| test.py:4:15:4:19 | Str | hi |
| test.py:4:23:4:27 | Str | \n |

4
python/ql/test/2/extractor-tests/multibyte/Test.ql Normal file
View File

@@ -0,0 +1,4 @@
import python
from StrConst s
select s, s.getText()

1
python/ql/test/2/extractor-tests/multibyte/options Normal file
View File

@@ -0,0 +1 @@
automatic_locations: true

4
python/ql/test/2/extractor-tests/multibyte/test.py Normal file
View File

@@ -0,0 +1,4 @@
#coding=utf8
b"đłe¶ŋ¶łŋ" + b"hi" + b"\n"
u"đłe¶ŋ¶łŋ" + u"hi" + u"\n"

7
python/ql/test/2/extractor-tests/normalise/Numbers.expected Normal file
View File

@@ -0,0 +1,7 @@
| int 0 | 0 |
| int 1 | 1 |
| int 2 | 2 |
| int 123745 | 123745 |
| int 268435455 | 268435455 |
| int 17216961135462248174 | 17216961135462248174 |
| int 100000000000000000000000000000000000000000000 | 100000000000000000000000000000000000000000000 |

16
python/ql/test/2/extractor-tests/normalise/Numbers.ql Normal file
View File

@@ -0,0 +1,16 @@
/**
* Much of the QL library handling integral values assumes that
* equivalence of the DB entities implies equivalence of the
* Python object and vice-versa.
* In Python 2, 1L == 1, which can cause problems, so we
* normalise all longs to ints.
*/
import python
from NumericObject n
where
exists(IntegerLiteral i | i.getLiteralObject() = n |
i.getEnclosingModule().getFile().getShortName() = "test.py"
)
select n.toString(), n.repr()

23
python/ql/test/2/extractor-tests/normalise/test.py Normal file
View File

@@ -0,0 +1,23 @@
#ints
0
0L
1
1L
2
2L
123745
123745L
0xfffffff
0xfffffffL
#Avoid values between 32 and 64 bits, as long is 32 bits on windows and 64 bits on 64 bit linux machines.
#longs
0xeeeeeeeeeeeeeeee
0xeeeeeeeeeeeeeeeeL
100000000000000000000000000000000000000000000
100000000000000000000000000000000000000000000L

1
python/ql/test/2/extractor-tests/object_hash/Success.expected Normal file
View File

@@ -0,0 +1 @@
| 1 |

4
python/ql/test/2/extractor-tests/object_hash/Success.ql Normal file
View File

@@ -0,0 +1,4 @@
import python
where exists(theSysModuleObject())
select 1

1
python/ql/test/2/extractor-tests/object_hash/test.py Normal file
View File

@@ -0,0 +1 @@
import cStringIO

1
python/ql/test/2/extractor-tests/old_style_disequality/test.expected Normal file
View File

@@ -0,0 +1 @@
| test.py:1:1:1:6 | Compare |

1
python/ql/test/2/extractor-tests/old_style_disequality/test.py Normal file
View File

@@ -0,0 +1 @@
1 <> 2

3
python/ql/test/2/extractor-tests/old_style_disequality/test.ql Normal file
View File

@@ -0,0 +1,3 @@
import python
select any(Compare o)

1
python/ql/test/2/extractor-tests/options Normal file
View File

@@ -0,0 +1 @@
automatic_locations: true

1
python/ql/test/2/extractor-tests/syntax_error/Modules.expected Normal file
View File

@@ -0,0 +1 @@
| Module test | test |

4
python/ql/test/2/extractor-tests/syntax_error/Modules.ql Normal file
View File

@@ -0,0 +1,4 @@
import python
from Module m
select m.toString(), m.getName()

1
python/ql/test/2/extractor-tests/syntax_error/Test.expected Normal file
View File

@@ -0,0 +1 @@
| Syntax Error |

4
python/ql/test/2/extractor-tests/syntax_error/Test.ql Normal file
View File

@@ -0,0 +1,4 @@
import python
from SyntaxError s
select s.toString()

2
python/ql/test/2/extractor-tests/syntax_error/test.py Normal file
View File

@@ -0,0 +1,2 @@
Not python at all :(

35
python/ql/test/2/library-tests/locations2.7plus/Locations.expected Normal file
View File

@@ -0,0 +1,35 @@
| Call | 2 | 17 | 2 | 28 |
| DictComp | 2 | 1 | 2 | 29 |
| ExprStmt | 2 | 1 | 2 | 29 |
| ExprStmt | 2 | 2 | 2 | 4 |
| ExprStmt | 3 | 1 | 3 | 14 |
| ExprStmt | 4 | 1 | 4 | 16 |
| ExprStmt | 4 | 3 | 4 | 3 |
| FastLocalsFunction | 2 | 1 | 2 | 29 |
| FastLocalsFunction | 4 | 1 | 4 | 16 |
| For | 2 | 1 | 2 | 29 |
| For | 4 | 1 | 4 | 16 |
| FunctionMetrics | 2 | 1 | 2 | 29 |
| FunctionMetrics | 4 | 1 | 4 | 16 |
| IntegerLiteral | 3 | 3 | 3 | 3 |
| IntegerLiteral | 3 | 6 | 3 | 6 |
| IntegerLiteral | 3 | 9 | 3 | 9 |
| IntegerLiteral | 3 | 12 | 3 | 12 |
| ModuleMetrics | 0 | 0 | 0 | 0 |
| Name | 2 | 1 | 2 | 29 |
| Name | 2 | 2 | 2 | 2 |
| Name | 2 | 4 | 2 | 4 |
| Name | 2 | 10 | 2 | 10 |
| Name | 2 | 12 | 2 | 12 |
| Name | 2 | 17 | 2 | 25 |
| Name | 2 | 27 | 2 | 27 |
| Name | 4 | 1 | 4 | 16 |
| Name | 4 | 3 | 4 | 3 |
| Name | 4 | 9 | 4 | 9 |
| Name | 4 | 14 | 4 | 14 |
| Set | 3 | 1 | 3 | 14 |
| SetComp | 4 | 1 | 4 | 16 |
| Tuple | 2 | 2 | 2 | 4 |
| Tuple | 2 | 10 | 2 | 12 |
| Yield | 2 | 2 | 2 | 4 |
| Yield | 4 | 3 | 4 | 3 |

5
python/ql/test/2/library-tests/locations2.7plus/Locations.ql Normal file
View File

@@ -0,0 +1,5 @@
import python
from AstNode ast, Location l
where ast.getLocation() = l
select ast.getAQlClass(), l.getStartLine(), l.getStartColumn(), l.getEndLine(), l.getEndColumn()

5
python/ql/test/2/library-tests/locations2.7plus/test.py Normal file
View File

@@ -0,0 +1,5 @@
{k:v for k,v in enumerate(y)}
{ 1, 2, 3, 4 }
{ i for i in y }

1
python/ql/test/3/extractor-tests/Kannada/Test.expected Normal file
View File

@@ -0,0 +1 @@
| Script \u0ca8\u0ca8\u0ccd\u0ca8_\u0cb8\u0ccd\u0c95\u0ccd\u0cb0\u0cbf\u0caa\u0ccd\u0c9f\u0ccd.py |

4
python/ql/test/3/extractor-tests/Kannada/Test.ql Normal file
View File

@@ -0,0 +1,4 @@
import python
from Module m
select m.toString()

9
python/ql/test/3/extractor-tests/Kannada/ನನ್ನ_ಸ್ಕ್ರಿಪ್ಟ್.py Normal file
View File

@@ -0,0 +1,9 @@
# -*- coding: utf-8 -*-
import sys
from dbgimporter import import_and_enable_debugger
import_and_enable_debugger()
def ಏನಾದರೂ_ಮಾಡು():
print('ಏನೋ ಮಾಡಿದೆ'.encode(sys.stdout.encoding, errors='replace'))
ಏನಾದರೂ_ಮಾಡು()

8
python/ql/test/3/extractor-tests/annotations/Assign.expected Normal file
View File

@@ -0,0 +1,8 @@
| 1 | AnnAssign | x | int | IntegerLiteral |
| 2 | AnnAssign | y | Thing | something() |
| 3 | AnnAssign | z | Any | ---- |
| 4 | AnnAssign | Attribute | complex | ---- |
| 5 | AnnAssign | Attribute | not_simple | None |
| 8 | AnnAssign | a | int | IntegerLiteral |
| 9 | AnnAssign | b | Thing | something() |
| 10 | AnnAssign | c | Any | ---- |

9
python/ql/test/3/extractor-tests/annotations/Assign.ql Normal file
View File

@@ -0,0 +1,9 @@
import python
from AnnAssign a, string value
where
value = a.getValue().toString()
or
not exists(a.getValue()) and value = "----"
select a.getLocation().getStartLine(), a.toString(), a.getTarget().toString(),
a.getAnnotation().toString(), value

38
python/ql/test/3/extractor-tests/annotations/Locations.expected Normal file
View File

@@ -0,0 +1,38 @@
| 0 | 0 | 0 | 0 | Module test |
| 1 | 1 | 1 | 1 | x |
| 1 | 1 | 1 | 10 | AnnAssign |
| 1 | 4 | 1 | 6 | int |
| 1 | 10 | 1 | 10 | IntegerLiteral |
| 2 | 1 | 2 | 1 | y |
| 2 | 1 | 2 | 22 | AnnAssign |
| 2 | 4 | 2 | 8 | Thing |
| 2 | 12 | 2 | 20 | something |
| 2 | 12 | 2 | 22 | something() |
| 3 | 1 | 3 | 1 | z |
| 3 | 1 | 3 | 6 | AnnAssign |
| 3 | 4 | 3 | 6 | Any |
| 4 | 1 | 4 | 1 | a |
| 4 | 1 | 4 | 3 | Attribute |
| 4 | 1 | 4 | 13 | AnnAssign |
| 4 | 7 | 4 | 13 | complex |
| 5 | 1 | 5 | 1 | c |
| 5 | 1 | 5 | 3 | Attribute |
| 5 | 1 | 5 | 23 | AnnAssign |
| 5 | 7 | 5 | 16 | not_simple |
| 5 | 20 | 5 | 23 | None |
| 7 | 1 | 7 | 8 | Function f |
| 7 | 1 | 7 | 8 | FunctionDef |
| 7 | 1 | 7 | 8 | FunctionExpr |
| 7 | 5 | 7 | 5 | f |
| 8 | 5 | 8 | 5 | a |
| 8 | 5 | 8 | 14 | AnnAssign |
| 8 | 8 | 8 | 10 | int |
| 8 | 14 | 8 | 14 | IntegerLiteral |
| 9 | 5 | 9 | 5 | b |
| 9 | 5 | 9 | 26 | AnnAssign |
| 9 | 8 | 9 | 12 | Thing |
| 9 | 16 | 9 | 24 | something |
| 9 | 16 | 9 | 26 | something() |
| 10 | 5 | 10 | 5 | c |
| 10 | 5 | 10 | 10 | AnnAssign |
| 10 | 8 | 10 | 10 | Any |

5
python/ql/test/3/extractor-tests/annotations/Locations.ql Normal file
View File

@@ -0,0 +1,5 @@
import python
from AstNode a, Location l
where l = a.getLocation()
select l.getStartLine(), l.getStartColumn(), l.getEndLine(), l.getEndColumn(), a.toString()

34
python/ql/test/3/extractor-tests/annotations/Successors.expected Normal file
View File

@@ -0,0 +1,34 @@
| 0 | Module test | 1 | IntegerLiteral |
| 1 | AnnAssign | 2 | something |
| 1 | IntegerLiteral | 1 | x |
| 1 | int | 1 | AnnAssign |
| 1 | x | 1 | int |
| 2 | AnnAssign | 3 | z |
| 2 | Thing | 2 | AnnAssign |
| 2 | something | 2 | something() |
| 2 | something() | 2 | y |
| 2 | y | 2 | Thing |
| 3 | AnnAssign | 4 | a |
| 3 | Any | 3 | AnnAssign |
| 3 | z | 3 | Any |
| 4 | AnnAssign | 5 | None |
| 4 | Attribute | 4 | complex |
| 4 | a | 4 | Attribute |
| 4 | complex | 4 | AnnAssign |
| 5 | AnnAssign | 7 | FunctionExpr |
| 5 | Attribute | 5 | not_simple |
| 5 | None | 5 | c |
| 5 | c | 5 | Attribute |
| 5 | not_simple | 5 | AnnAssign |
| 7 | Function f | 8 | IntegerLiteral |
| 7 | FunctionExpr | 7 | f |
| 7 | f | 0 | Module test |
| 8 | AnnAssign | 9 | something |
| 8 | IntegerLiteral | 8 | a |
| 8 | a | 8 | AnnAssign |
| 9 | AnnAssign | 10 | c |
| 9 | b | 9 | AnnAssign |
| 9 | something | 9 | something() |
| 9 | something() | 9 | b |
| 10 | AnnAssign | 7 | Function f |
| 10 | c | 10 | AnnAssign |

6
python/ql/test/3/extractor-tests/annotations/Successors.ql Normal file
View File

@@ -0,0 +1,6 @@
import python
from ControlFlowNode p, ControlFlowNode s
where p.getASuccessor() = s
select p.getLocation().getStartLine(), p.getNode().toString(), s.getLocation().getStartLine(),
s.getNode().toString()

10
python/ql/test/3/extractor-tests/annotations/test.py Normal file
View File

@@ -0,0 +1,10 @@
x: int = 0
y: Thing = something()
z: Any
a.x : complex
c.y : not_simple = None
def f():
a: int = 0
b: Thing = something()
c: Any

1
python/ql/test/3/extractor-tests/async3.5/AsyncFor.expected Normal file
View File

@@ -0,0 +1 @@
| 3 | For |

4
python/ql/test/3/extractor-tests/async3.5/AsyncFor.ql Normal file
View File

@@ -0,0 +1,4 @@
import python
from AsyncFor a
select a.getLocation().getStartLine(), a.toString()

1
python/ql/test/3/extractor-tests/async3.5/AsyncWith.expected Normal file
View File

@@ -0,0 +1 @@
| 4 | With |

4
python/ql/test/3/extractor-tests/async3.5/AsyncWith.ql Normal file
View File

@@ -0,0 +1,4 @@
import python
from AsyncWith aw
select aw.getLocation().getStartLine(), aw.toString()

1
python/ql/test/3/extractor-tests/async3.5/Await.expected Normal file
View File

@@ -0,0 +1 @@
| 6 | Await |

4
python/ql/test/3/extractor-tests/async3.5/Await.ql Normal file
View File

@@ -0,0 +1,4 @@
import python
from Await a
select a.getLocation().getStartLine(), a.toString()

8
python/ql/test/3/extractor-tests/async3.5/test.py Normal file
View File

@@ -0,0 +1,8 @@
async def foo():
async for x in y:
async with a as b:
pass
await z

9
python/ql/test/3/extractor-tests/async3.6/AsyncFor.expected Normal file
View File

@@ -0,0 +1,9 @@
| 7 | For | normal |
| 14 | For | async |
| 15 | For | normal |
| 16 | For | async |
| 17 | For | normal |
| 22 | For | normal |
| 23 | For | normal |
| 24 | For | normal |
| 25 | For | normal |

8
python/ql/test/3/extractor-tests/async3.6/AsyncFor.ql Normal file
View File

@@ -0,0 +1,8 @@
import python
from For f, string kind
where
f instanceof AsyncFor and kind = "async"
or
not f instanceof AsyncFor and kind = "normal"
select f.getLocation().getStartLine(), f.toString(), kind

3
python/ql/test/3/extractor-tests/async3.6/Await.expected Normal file
View File

@@ -0,0 +1,3 @@
| 9 | Await |
| 15 | Await |
| 17 | Await |

4
python/ql/test/3/extractor-tests/async3.6/Await.ql Normal file
View File

@@ -0,0 +1,4 @@
import python
from Await a
select a.getLocation().getStartLine(), a.toString()

Some files were not shown because too many files have changed in this diff Show More