hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2026-04-25 08:45:14 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into python/test-MaD-keyword-argument

Browse Source
This commit is contained in:
yoff
2024-03-22 10:56:08 +01:00
committed by GitHub
parent eef60c9ad2 fb4ed39d89
commit c520cb6d58
862 changed files with 70983 additions and 70154 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
python/BUILD.bazel
View File

@@ -1,4 +1,6 @@
load("@rules_pkg//:mappings.bzl", "pkg_filegroup", "pkg_files")
load("@rules_pkg//pkg:mappings.bzl", "pkg_filegroup", "pkg_files")
load("@semmle_code//:dist.bzl", "dist", "pack_zip")
load("//:defs.bzl", "codeql_platform")
package(default_visibility = ["//visibility:public"])
@@ -28,3 +30,32 @@ pkg_filegroup(
"//python/downgrades",
],
)
pkg_files(
name = "codeql-extractor-yml",
srcs = ["codeql-extractor.yml"],
strip_prefix = None,
)
dist(
name = "extractor-generic",
srcs = [
":codeql-extractor-yml",
":dbscheme-group",
"//python/downgrades",
"//python/extractor",
"//python/tools",
],
prefix = "python",
visibility = ["//visibility:public"],
)
pack_zip(
name = "extractor-arch",
srcs = [
"//python/extractor/tsg-python",
],
package_file_name = "extractor-" + codeql_platform + ".zip",
prefix = "python/tools/" + codeql_platform,
visibility = ["//visibility:public"],
)

46
python/codeql-extractor.yml Normal file
View File

@@ -0,0 +1,46 @@
name: "python"
display_name: "Python"
version: 1.22.1
column_kind: utf32
build_modes:
- none
github_api_languages:
- Python
scc_languages:
- Python
file_types:
- name: python
display_name: Python sources
extensions:
- .py
legacy_qltest_extraction: true
options:
logging:
title: Options pertaining to logging.
type: object
properties:
verbosity:
title: Python extractor logging verbosity level.
description: >
Controls the level of verbosity of the CodeQL Python extractor.
The supported levels are (in order of increasing verbosity):
- off
- errors
- warnings
- info or progress
- debug or progress+
- trace or progress++
- progress+++
type: string
pattern: "^(off|errors|warnings|(info|progress)|(debug|progress\\+)|(trace|progress\\+\\+)|progress\\+\\+\\+)$"
python_executable_name:
title: Controls the name of the Python executable used by the Python extractor.
description: >
The Python extractor uses platform-dependent heuristics to determine the name of the Python executable to use.
Specifying a value for this option overrides the name of the Python executable used by the extractor.
Accepted values are py, python and python3.
Use this setting with caution, the Python extractor requires Python 3 to run.
type: string
pattern: "^(py|python|python3)$"

4
python/extractor/BUILD.bazel
View File

@@ -1,4 +1,4 @@
load("//:dist.bzl", "pack_zip")
load("@semmle_code//:dist.bzl", "pack_zip")
py_binary(
name = "make-zips-py",
@@ -33,7 +33,7 @@ genrule(
)
pack_zip(
name = "extractor-python",
name = "extractor",
srcs = [
"LICENSE-PSF.md", # because we distribute imp.py
"convert_setup.py",

2
python/extractor/cli-integration-test/writing-diagnostics/test_diagnostics_output.py
View File

@@ -1,6 +1,6 @@
import os
import sys
sys.path.append(os.path.join(os.path.dirname(__file__), "..", "..", "..", "integration-tests"))
sys.path.append(os.path.join(os.path.dirname(__file__), "..", "..", "..", "..", "..", "integration-tests"))
import diagnostics_test_utils
test_db = "db"

4
python/extractor/qlpack.yml
View File

@@ -1,6 +1,6 @@
name: extractor-python
dependencies:
codeql/python-all: "*"
codeql/python-queries: "*"
codeql/python-all: ${workspace}
codeql/python-queries: ${workspace}
extractor: python
warnOnImplicitThis: true

2
python/extractor/tests/test_patterns.py
View File

@@ -12,7 +12,7 @@ class ExtractorPatternsTest(test_utils.ExtractorTest):
def test(self):
repo_dir = subprocess.Popen(["git", "rev-parse", "--show-toplevel"], stdout=subprocess.PIPE).communicate()[0].rstrip().decode("utf-8")
test_file_path = os.path.abspath(os.path.join(repo_dir, "unit-tests", "files", "pattern-matching", "patterns.json"))
test_file_path = os.path.abspath(os.path.join(repo_dir, "..", "unit-tests", "files", "pattern-matching", "patterns.json"))
with open(test_file_path) as test_file:
test_patterns = json.load(test_file)
for test_pattern in test_patterns:

6
python/extractor/tsg-python/BUILD.bazel
View File

@@ -1,5 +1,5 @@
load("@tsg_python_crate_index//:defs.bzl", "aliases", "all_crate_deps")
load("//:common.bzl", "codeql_rust_binary")
load("@py_deps//:defs.bzl", "aliases", "all_crate_deps")
load("@semmle_code//:common.bzl", "codeql_rust_binary")
codeql_rust_binary(
name = "tsg-python",
@@ -12,5 +12,5 @@ codeql_rust_binary(
visibility = ["//visibility:public"],
deps = all_crate_deps(
normal = True,
) + ["//extractor-python/tsg-python/tree-sitter-python"],
) + ["//python/extractor/tsg-python/tsp"],
)

212
python/extractor/tsg-python/Cargo.Bazel.lock
View File

@@ -1,5 +1,5 @@
{
"checksum": "54f1095f5a2e74da736682bc8d355b3dbce47558983feba04faba84cf3abfaca",
"checksum": "35a1ce4b6c4f997c496c11d3a8fcfaadc5833dfd41bebb022941687d73dde159",
"crates": {
"ahash 0.4.7": {
"name": "ahash",
@@ -7,7 +7,7 @@
"package_url": "https://github.com/tkaitchuck/ahash",
"repository": {
"Http": {
"url": "https://crates.io/api/v1/crates/ahash/0.4.7/download",
"url": "https://static.crates.io/crates/ahash/0.4.7/download",
"sha256": "739f4a8db6605981345c5654f3a85b056ce52f37a39d34da03f25bf2151ea16e"
}
},
@@ -43,7 +43,7 @@
"package_url": "https://github.com/BurntSushi/aho-corasick",
"repository": {
"Http": {
"url": "https://crates.io/api/v1/crates/aho-corasick/0.7.18/download",
"url": "https://static.crates.io/crates/aho-corasick/0.7.18/download",
"sha256": "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f"
}
},
@@ -95,7 +95,7 @@
"package_url": "https://github.com/ogham/rust-ansi-term",
"repository": {
"Http": {
"url": "https://crates.io/api/v1/crates/ansi_term/0.11.0/download",
"url": "https://static.crates.io/crates/ansi_term/0.11.0/download",
"sha256": "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b"
}
},
@@ -141,7 +141,7 @@
"package_url": "https://github.com/dtolnay/anyhow",
"repository": {
"Http": {
"url": "https://crates.io/api/v1/crates/anyhow/1.0.44/download",
"url": "https://static.crates.io/crates/anyhow/1.0.44/download",
"sha256": "61604a8f862e1d5c3229fdd78f8b02c68dcf73a4c4b05fd636d12240aaa242c1"
}
},
@@ -207,7 +207,7 @@
"package_url": "https://github.com/softprops/atty",
"repository": {
"Http": {
"url": "https://crates.io/api/v1/crates/atty/0.2.14/download",
"url": "https://static.crates.io/crates/atty/0.2.14/download",
"sha256": "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
}
},
@@ -265,7 +265,7 @@
"package_url": "https://github.com/bitflags/bitflags",
"repository": {
"Http": {
"url": "https://crates.io/api/v1/crates/bitflags/1.3.2/download",
"url": "https://static.crates.io/crates/bitflags/1.3.2/download",
"sha256": "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
}
},
@@ -307,7 +307,7 @@
"package_url": "https://github.com/alexcrichton/cc-rs",
"repository": {
"Http": {
"url": "https://crates.io/api/v1/crates/cc/1.0.70/download",
"url": "https://static.crates.io/crates/cc/1.0.70/download",
"sha256": "d26a6ce4b6a484fa3edb70f7efa6fc430fd2b87285fe8b84304fd0936faa0dc0"
}
},
@@ -343,7 +343,7 @@
"package_url": "https://github.com/alexcrichton/cfg-if",
"repository": {
"Http": {
"url": "https://crates.io/api/v1/crates/cfg-if/1.0.0/download",
"url": "https://static.crates.io/crates/cfg-if/1.0.0/download",
"sha256": "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
}
},
@@ -379,7 +379,7 @@
"package_url": "https://github.com/clap-rs/clap",
"repository": {
"Http": {
"url": "https://crates.io/api/v1/crates/clap/2.33.3/download",
"url": "https://static.crates.io/crates/clap/2.33.3/download",
"sha256": "37e58ac78573c40708d45522f0d80fa2f01cc4f9b4e2bf749807255454312002"
}
},
@@ -462,7 +462,7 @@
"package_url": "https://github.com/rust-lang/hashbrown",
"repository": {
"Http": {
"url": "https://crates.io/api/v1/crates/hashbrown/0.9.1/download",
"url": "https://static.crates.io/crates/hashbrown/0.9.1/download",
"sha256": "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04"
}
},
@@ -514,7 +514,7 @@
"package_url": "https://github.com/hermitcore/libhermit-rs",
"repository": {
"Http": {
"url": "https://crates.io/api/v1/crates/hermit-abi/0.1.19/download",
"url": "https://static.crates.io/crates/hermit-abi/0.1.19/download",
"sha256": "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"
}
},
@@ -559,7 +559,7 @@
"package_url": "https://github.com/dtolnay/itoa",
"repository": {
"Http": {
"url": "https://crates.io/api/v1/crates/itoa/1.0.1/download",
"url": "https://static.crates.io/crates/itoa/1.0.1/download",
"sha256": "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35"
}
},
@@ -595,7 +595,7 @@
"package_url": "https://github.com/rust-lang/libc",
"repository": {
"Http": {
"url": "https://crates.io/api/v1/crates/libc/0.2.101/download",
"url": "https://static.crates.io/crates/libc/0.2.101/download",
"sha256": "3cb00336871be5ed2c8ed44b60ae9959dc5b9f08539422ed43f09e34ecaeba21"
}
},
@@ -654,7 +654,7 @@
"package_url": "https://github.com/rust-lang/log",
"repository": {
"Http": {
"url": "https://crates.io/api/v1/crates/log/0.4.14/download",
"url": "https://static.crates.io/crates/log/0.4.14/download",
"sha256": "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710"
}
},
@@ -717,7 +717,7 @@
"package_url": "https://github.com/BurntSushi/memchr",
"repository": {
"Http": {
"url": "https://crates.io/api/v1/crates/memchr/2.4.1/download",
"url": "https://static.crates.io/crates/memchr/2.4.1/download",
"sha256": "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a"
}
},
@@ -783,7 +783,7 @@
"package_url": "https://github.com/alexcrichton/proc-macro2",
"repository": {
"Http": {
"url": "https://crates.io/api/v1/crates/proc-macro2/1.0.29/download",
"url": "https://static.crates.io/crates/proc-macro2/1.0.29/download",
"sha256": "b9f5105d4fdaab20335ca9565e106a5d9b82b6219b5ba735731124ac6711d23d"
}
},
@@ -853,7 +853,7 @@
"package_url": "https://github.com/dtolnay/quote",
"repository": {
"Http": {
"url": "https://crates.io/api/v1/crates/quote/1.0.9/download",
"url": "https://static.crates.io/crates/quote/1.0.9/download",
"sha256": "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7"
}
},
@@ -905,7 +905,7 @@
"package_url": "https://github.com/rust-lang/regex",
"repository": {
"Http": {
"url": "https://crates.io/api/v1/crates/regex/1.5.5/download",
"url": "https://static.crates.io/crates/regex/1.5.5/download",
"sha256": "1a11647b6b25ff05a515cb92c365cec08801e83423a235b51e231e1808747286"
}
},
@@ -980,7 +980,7 @@
"package_url": "https://github.com/rust-lang/regex",
"repository": {
"Http": {
"url": "https://crates.io/api/v1/crates/regex-syntax/0.6.25/download",
"url": "https://static.crates.io/crates/regex-syntax/0.6.25/download",
"sha256": "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b"
}
},
@@ -1030,7 +1030,7 @@
"package_url": "https://github.com/dtolnay/ryu",
"repository": {
"Http": {
"url": "https://crates.io/api/v1/crates/ryu/1.0.9/download",
"url": "https://static.crates.io/crates/ryu/1.0.9/download",
"sha256": "73b4b750c782965c211b42f022f59af1fbceabdd026623714f104152f1ec149f"
}
},
@@ -1066,7 +1066,7 @@
"package_url": "https://github.com/serde-rs/serde",
"repository": {
"Http": {
"url": "https://crates.io/api/v1/crates/serde/1.0.136/download",
"url": "https://static.crates.io/crates/serde/1.0.136/download",
"sha256": "ce31e24b01e1e524df96f1c2fdd054405f8d7376249a5110886fb4b658484789"
}
},
@@ -1132,7 +1132,7 @@
"package_url": "https://github.com/serde-rs/json",
"repository": {
"Http": {
"url": "https://crates.io/api/v1/crates/serde_json/1.0.79/download",
"url": "https://static.crates.io/crates/serde_json/1.0.79/download",
"sha256": "8e8d9fa5c3b304765ce1fd9c4c8a3de2c8db365a5b91be52f186efc675681d95"
}
},
@@ -1210,7 +1210,7 @@
"package_url": "https://github.com/servo/rust-smallvec",
"repository": {
"Http": {
"url": "https://crates.io/api/v1/crates/smallvec/1.6.1/download",
"url": "https://static.crates.io/crates/smallvec/1.6.1/download",
"sha256": "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e"
}
},
@@ -1252,7 +1252,7 @@
"package_url": "https://github.com/robbepop/string-interner",
"repository": {
"Http": {
"url": "https://crates.io/api/v1/crates/string-interner/0.12.2/download",
"url": "https://static.crates.io/crates/string-interner/0.12.2/download",
"sha256": "383196d1876517ee6f9f0864d1fc1070331b803335d3c6daaa04bbcccd823c08"
}
},
@@ -1309,7 +1309,7 @@
"package_url": "https://github.com/dguo/strsim-rs",
"repository": {
"Http": {
"url": "https://crates.io/api/v1/crates/strsim/0.8.0/download",
"url": "https://static.crates.io/crates/strsim/0.8.0/download",
"sha256": "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"
}
},
@@ -1344,7 +1344,7 @@
"package_url": "https://github.com/dtolnay/syn",
"repository": {
"Http": {
"url": "https://crates.io/api/v1/crates/syn/1.0.76/download",
"url": "https://static.crates.io/crates/syn/1.0.76/download",
"sha256": "c6f107db402c2c2055242dbf4d2af0e69197202e9faacbef9571bbe47f5a1b84"
}
},
@@ -1427,7 +1427,7 @@
"package_url": "https://github.com/mgeisler/textwrap",
"repository": {
"Http": {
"url": "https://crates.io/api/v1/crates/textwrap/0.11.0/download",
"url": "https://static.crates.io/crates/textwrap/0.11.0/download",
"sha256": "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060"
}
},
@@ -1471,7 +1471,7 @@
"package_url": "https://github.com/dtolnay/thiserror",
"repository": {
"Http": {
"url": "https://crates.io/api/v1/crates/thiserror/1.0.29/download",
"url": "https://static.crates.io/crates/thiserror/1.0.29/download",
"sha256": "602eca064b2d83369e2b2f34b09c70b605402801927c65c11071ac911d299b88"
}
},
@@ -1516,7 +1516,7 @@
"package_url": "https://github.com/dtolnay/thiserror",
"repository": {
"Http": {
"url": "https://crates.io/api/v1/crates/thiserror-impl/1.0.29/download",
"url": "https://static.crates.io/crates/thiserror-impl/1.0.29/download",
"sha256": "bad553cc2c78e8de258400763a647e80e6d1b31ee237275d756f6836d204494c"
}
},
@@ -1569,7 +1569,7 @@
"package_url": "https://github.com/tree-sitter/tree-sitter",
"repository": {
"Http": {
"url": "https://crates.io/api/v1/crates/tree-sitter/0.20.4/download",
"url": "https://static.crates.io/crates/tree-sitter/0.20.4/download",
"sha256": "4e34327f8eac545e3f037382471b2b19367725a242bba7bc45edb9efb49fe39a"
}
},
@@ -1640,7 +1640,7 @@
"package_url": "https://github.com/tree-sitter/tree-sitter-graph/",
"repository": {
"Http": {
"url": "https://crates.io/api/v1/crates/tree-sitter-graph/0.7.0/download",
"url": "https://static.crates.io/crates/tree-sitter-graph/0.7.0/download",
"sha256": "639d21e886f581d293de5f5081f09af003c54607ff3fa85efa159b243ba1f97a"
}
},
@@ -1707,72 +1707,6 @@
],
"license_file": null
},
"tree-sitter-python 0.19.0": {
"name": "tree-sitter-python",
"version": "0.19.0",
"package_url": "https://github.com/tree-sitter/tree-sitter-python",
"repository": null,
"targets": [
{
"Library": {
"crate_name": "tree_sitter_python",
"crate_root": "bindings/rust/lib.rs",
"srcs": [
"**/*.rs"
]
}
},
{
"BuildScript": {
"crate_name": "build_script_build",
"crate_root": "bindings/rust/build.rs",
"srcs": [
"**/*.rs"
]
}
}
],
"library_target_name": "tree_sitter_python",
"common_attrs": {
"compile_data_glob": [
"**"
],
"deps": {
"common": [
{
"id": "tree-sitter 0.20.4",
"target": "tree_sitter"
},
{
"id": "tree-sitter-python 0.19.0",
"target": "build_script_build"
}
],
"selects": {}
},
"edition": "2018",
"version": "0.19.0"
},
"build_script_attrs": {
"data_glob": [
"**"
],
"deps": {
"common": [
{
"id": "cc 1.0.70",
"target": "cc"
}
],
"selects": {}
}
},
"license": "MIT",
"license_ids": [
"MIT"
],
"license_file": null
},
"tsg-python 0.1.0": {
"name": "tsg-python",
"version": "0.1.0",
@@ -1828,13 +1762,79 @@
"license_ids": [],
"license_file": null
},
"tsp 0.19.0": {
"name": "tsp",
"version": "0.19.0",
"package_url": "https://github.com/tree-sitter/tree-sitter-python",
"repository": null,
"targets": [
{
"Library": {
"crate_name": "tsp",
"crate_root": "bindings/rust/lib.rs",
"srcs": [
"**/*.rs"
]
}
},
{
"BuildScript": {
"crate_name": "build_script_build",
"crate_root": "bindings/rust/build.rs",
"srcs": [
"**/*.rs"
]
}
}
],
"library_target_name": "tsp",
"common_attrs": {
"compile_data_glob": [
"**"
],
"deps": {
"common": [
{
"id": "tree-sitter 0.20.4",
"target": "tree_sitter"
},
{
"id": "tsp 0.19.0",
"target": "build_script_build"
}
],
"selects": {}
},
"edition": "2018",
"version": "0.19.0"
},
"build_script_attrs": {
"data_glob": [
"**"
],
"deps": {
"common": [
{
"id": "cc 1.0.70",
"target": "cc"
}
],
"selects": {}
}
},
"license": "MIT",
"license_ids": [
"MIT"
],
"license_file": null
},
"unicode-width 0.1.8": {
"name": "unicode-width",
"version": "0.1.8",
"package_url": "https://github.com/unicode-rs/unicode-width",
"repository": {
"Http": {
"url": "https://crates.io/api/v1/crates/unicode-width/0.1.8/download",
"url": "https://static.crates.io/crates/unicode-width/0.1.8/download",
"sha256": "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3"
}
},
@@ -1876,7 +1876,7 @@
"package_url": "https://github.com/unicode-rs/unicode-xid",
"repository": {
"Http": {
"url": "https://crates.io/api/v1/crates/unicode-xid/0.2.2/download",
"url": "https://static.crates.io/crates/unicode-xid/0.2.2/download",
"sha256": "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3"
}
},
@@ -1918,7 +1918,7 @@
"package_url": "https://github.com/contain-rs/vec-map",
"repository": {
"Http": {
"url": "https://crates.io/api/v1/crates/vec_map/0.8.2/download",
"url": "https://static.crates.io/crates/vec_map/0.8.2/download",
"sha256": "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191"
}
},
@@ -1954,7 +1954,7 @@
"package_url": "https://github.com/retep998/winapi-rs",
"repository": {
"Http": {
"url": "https://crates.io/api/v1/crates/winapi/0.3.9/download",
"url": "https://static.crates.io/crates/winapi/0.3.9/download",
"sha256": "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
}
},
@@ -2037,7 +2037,7 @@
"package_url": "https://github.com/retep998/winapi-rs",
"repository": {
"Http": {
"url": "https://crates.io/api/v1/crates/winapi-i686-pc-windows-gnu/0.4.0/download",
"url": "https://static.crates.io/crates/winapi-i686-pc-windows-gnu/0.4.0/download",
"sha256": "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
}
},
@@ -2096,7 +2096,7 @@
"package_url": "https://github.com/retep998/winapi-rs",
"repository": {
"Http": {
"url": "https://crates.io/api/v1/crates/winapi-x86_64-pc-windows-gnu/0.4.0/download",
"url": "https://static.crates.io/crates/winapi-x86_64-pc-windows-gnu/0.4.0/download",
"sha256": "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
}
},
@@ -2152,8 +2152,8 @@
},
"binary_crates": [],
"workspace_members": {
"tree-sitter-python 0.19.0": "extractor-python/tsg-python/tree-sitter-python",
"tsg-python 0.1.0": "extractor-python/tsg-python"
"tsg-python 0.1.0": "python/extractor/tsg-python",
"tsp 0.19.0": "python/extractor/tsg-python/tsp"
},
"conditions": {
"aarch64-apple-darwin": [

18
python/extractor/tsg-python/Cargo.lock generated
View File

@@ -267,14 +267,6 @@ dependencies = [
"tree-sitter",
]
[[package]]
name = "tree-sitter-python"
version = "0.19.0"
dependencies = [
"cc",
"tree-sitter",
]
[[package]]
name = "tsg-python"
version = "0.1.0"
@@ -287,7 +279,15 @@ dependencies = [
"thiserror",
"tree-sitter",
"tree-sitter-graph",
"tree-sitter-python",
"tsp",
]
[[package]]
name = "tsp"
version = "0.19.0"
dependencies = [
"cc",
"tree-sitter",
]
[[package]]

4
python/extractor/tsg-python/Cargo.toml
View File

@@ -8,7 +8,7 @@ edition = "2018"
# When changing/updating these, the `Cargo.Bazel.lock` file has to be regenerated.
# Check out the documentation at https://bazelbuild.github.io/rules_rust/crate_universe.html#repinning--updating-dependencies
# for how to do so. The bazel repository for the tsg-python project is called `tsg_python_crate_index`,
# for how to do so. The bazel repository for the tsg-python project is called `py_deps`,
# and instead of calling `bazel sync`, `./build --bazel sync` should be used instead, to always use the correct bazel version.
[dependencies]
anyhow = "1.0"
@@ -17,7 +17,7 @@ smallvec = { version="1.6", features=["union"] }
thiserror = "1.0"
tree-sitter = "0.20.4"
tree-sitter-graph = "0.7.0"
tree-sitter-python = {path = "tree-sitter-python"}
tsp = {path = "tsp"}
clap = "2.32"
[dependencies.string-interner]

2
python/extractor/tsg-python/src/main.rs
View File

@@ -488,7 +488,7 @@ fn main() -> Result<()> {
"bundled `python.tsg`".to_owned()
};
let source_path = Path::new(matches.value_of("source").unwrap());
let language = tree_sitter_python::language();
let language = tsp::language();
let mut parser = Parser::new();
parser.set_language(language)?;
// Statically include `python.tsg`:

0
python/extractor/tsg-python/tree-sitter-python/.gitignore → python/extractor/tsg-python/tsp/.gitignore vendored
View File

0
python/extractor/tsg-python/tree-sitter-python/.npmignore → python/extractor/tsg-python/tsp/.npmignore
View File

8
python/extractor/tsg-python/tree-sitter-python/BUILD.bazel → python/extractor/tsg-python/tsp/BUILD.bazel
View File

@@ -1,13 +1,13 @@
load("@rules_rust//cargo:defs.bzl", "cargo_build_script")
load("@rules_rust//rust:defs.bzl", "rust_library")
load("@tsg_python_crate_index//:defs.bzl", "aliases", "all_crate_deps")
load("@py_deps//:defs.bzl", "aliases", "all_crate_deps")
package(default_visibility = ["//visibility:public"])
# This will run the build script from the root of the workspace, and
# collect the outputs.
cargo_build_script(
name = "tsg-build-script",
name = "tsg-build",
srcs = ["bindings/rust/build.rs"],
data = glob([
"src/**",
@@ -18,7 +18,7 @@ cargo_build_script(
)
rust_library(
name = "tree-sitter-python",
name = "tsp",
srcs = [
"bindings/rust/lib.rs",
],
@@ -32,7 +32,7 @@ rust_library(
proc_macro_deps = all_crate_deps(
proc_macro = True,
),
deps = [":tsg-build-script"] + all_crate_deps(
deps = [":tsg-build"] + all_crate_deps(
normal = True,
),
)

2
python/extractor/tsg-python/tree-sitter-python/Cargo.toml → python/extractor/tsg-python/tsp/Cargo.toml
View File

@@ -1,5 +1,5 @@
[package]
name = "tree-sitter-python"
name = "tsp"
description = "Python grammar for the tree-sitter parsing library"
version = "0.19.0"
authors = [

0
python/extractor/tsg-python/tree-sitter-python/LICENSE → python/extractor/tsg-python/tsp/LICENSE
View File

0
python/extractor/tsg-python/tree-sitter-python/README.md → python/extractor/tsg-python/tsp/README.md
View File

0
python/extractor/tsg-python/tree-sitter-python/binding.gyp → python/extractor/tsg-python/tsp/binding.gyp
View File

0
python/extractor/tsg-python/tree-sitter-python/bindings/node/binding.cc → python/extractor/tsg-python/tsp/bindings/node/binding.cc
View File

0
python/extractor/tsg-python/tree-sitter-python/bindings/node/index.js → python/extractor/tsg-python/tsp/bindings/node/index.js
View File

0
python/extractor/tsg-python/tree-sitter-python/bindings/rust/README.md → python/extractor/tsg-python/tsp/bindings/rust/README.md
View File

0
python/extractor/tsg-python/tree-sitter-python/bindings/rust/build.rs → python/extractor/tsg-python/tsp/bindings/rust/build.rs
View File

0
python/extractor/tsg-python/tree-sitter-python/bindings/rust/lib.rs → python/extractor/tsg-python/tsp/bindings/rust/lib.rs
View File

0
python/extractor/tsg-python/tree-sitter-python/grammar.js → python/extractor/tsg-python/tsp/grammar.js
View File

0
python/extractor/tsg-python/tree-sitter-python/log.html → python/extractor/tsg-python/tsp/log.html
View File

0
python/extractor/tsg-python/tree-sitter-python/package.json → python/extractor/tsg-python/tsp/package.json
View File

0
python/extractor/tsg-python/tree-sitter-python/queries/highlights.scm → python/extractor/tsg-python/tsp/queries/highlights.scm
View File

0
python/extractor/tsg-python/tree-sitter-python/queries/tags.scm → python/extractor/tsg-python/tsp/queries/tags.scm
View File

0
python/extractor/tsg-python/tree-sitter-python/src/grammar.json → python/extractor/tsg-python/tsp/src/grammar.json
View File

0
python/extractor/tsg-python/tree-sitter-python/src/node-types.json → python/extractor/tsg-python/tsp/src/node-types.json
View File

0
python/extractor/tsg-python/tree-sitter-python/src/parser.c → python/extractor/tsg-python/tsp/src/parser.c
View File

0
python/extractor/tsg-python/tree-sitter-python/src/scanner.cc → python/extractor/tsg-python/tsp/src/scanner.cc
View File

0
python/extractor/tsg-python/tree-sitter-python/src/tree_sitter/parser.h → python/extractor/tsg-python/tsp/src/tree_sitter/parser.h
View File

8
python/ql/consistency-queries/DataFlowConsistency.ql
View File

@@ -10,12 +10,16 @@ private import semmle.python.dataflow.new.internal.DataFlowDispatch
private import semmle.python.dataflow.new.internal.TaintTrackingImplSpecific
private import codeql.dataflow.internal.DataFlowImplConsistency
private module Input implements InputSig<PythonDataFlow> {
private module Input implements InputSig<Location, PythonDataFlow> {
private import Private
private import Public
predicate postWithInFlowExclude(Node n) { n instanceof FlowSummaryNode }
predicate uniqueNodeLocationExclude(Node n) { n instanceof FlowSummaryNode }
predicate missingLocationExclude(Node n) { n instanceof FlowSummaryNode }
predicate argHasPostUpdateExclude(ArgumentNode n) {
// TODO: Implement post-updates for *args, see tests added in https://github.com/github/codeql/pull/14936
exists(ArgumentPosition apos | n.argumentOf(_, apos) and apos.isStarArgs(_))
@@ -132,4 +136,4 @@ private module Input implements InputSig<PythonDataFlow> {
}
}
import MakeConsistency<PythonDataFlow, PythonTaintTracking, Input>
import MakeConsistency<Location, PythonDataFlow, PythonTaintTracking, Input>

42
python/ql/consistency-queries/TypeTrackingConsistency.ql Normal file
View File

@@ -0,0 +1,42 @@
private import python
private import semmle.python.dataflow.new.DataFlow
private import semmle.python.dataflow.new.internal.DataFlowPrivate as DataFlowPrivate
private import semmle.python.dataflow.new.internal.TypeTrackingImpl
private module ConsistencyChecksInput implements ConsistencyChecksInputSig {
predicate unreachableNodeExclude(DataFlow::Node n) {
n instanceof DataFlowPrivate::SyntheticPostUpdateNode
or
n instanceof DataFlowPrivate::SyntheticPreUpdateNode
or
// TODO: when adding support for proper content, handle **kwargs passing better!
n instanceof DataFlowPrivate::SynthDictSplatArgumentNode
or
// TODO: when adding support for proper content, handle unpacking tuples in match
// cases better, such as
//
// match (NONSOURCE, SOURCE):
// case (x, y): ...
exists(DataFlow::Node m |
m.asCfgNode().getNode() instanceof MatchCapturePattern
or
m.asCfgNode().getNode() instanceof MatchAsPattern
or
m.asCfgNode().getNode() instanceof MatchOrPattern
|
TypeTrackingInput::simpleLocalSmallStep*(m, n)
)
or
// TODO: when adding support for proper content, handle iterable unpacking better
// such as `for k,v in items:`, or `a, (b,c) = ...`
n instanceof DataFlow::IterableSequenceNode
or
// We have missing use-use flow in
// https://github.com/python/cpython/blob/0fb18b02c8ad56299d6a2910be0bab8ad601ef24/Lib/socketserver.py#L276-L303
// which I couldn't just fix. We ignore the problems here, and instead rely on the
// test-case added in https://github.com/github/codeql/pull/15841
n.getLocation().getFile().getAbsolutePath().matches("%/socketserver.py")
}
}
import ConsistencyChecks<ConsistencyChecksInput>

11
python/ql/lib/CHANGELOG.md
View File

@@ -1,3 +1,14 @@
## 0.11.11
No user-facing changes.
## 0.11.10
### Minor Analysis Improvements
* Fixed missing flow for dictionary updates (`d[<key>] = ...`) when `<key>` is a string constant not used in dictionary literals or as name of keyword-argument.
* Fixed flow for iterable unpacking (`a,b = my_tuple`) when it occurs on top-level (module) scope.
## 0.11.9
### Minor Analysis Improvements

4
python/ql/lib/change-notes/2024-02-28-iterable-unpacking-module-scope.md
View File

@@ -1,4 +0,0 @@
---
category: minorAnalysis
---
* Fixed flow for iterable unpacking (`a,b = my_tuple`) when it occurs on top-level (module) scope.

8
python/ql/lib/change-notes/2024-03-01-dict-update-content.md → python/ql/lib/change-notes/released/0.11.10.md
View File

@@ -1,4 +1,6 @@
---
category: minorAnalysis
---
## 0.11.10
### Minor Analysis Improvements
* Fixed missing flow for dictionary updates (`d[<key>] = ...`) when `<key>` is a string constant not used in dictionary literals or as name of keyword-argument.
* Fixed flow for iterable unpacking (`a,b = my_tuple`) when it occurs on top-level (module) scope.

3
python/ql/lib/change-notes/released/0.11.11.md Normal file
View File

@@ -0,0 +1,3 @@
## 0.11.11
No user-facing changes.

2
python/ql/lib/codeql-pack.release.yml
View File

@@ -1,2 +1,2 @@
---
lastReleaseVersion: 0.11.9
lastReleaseVersion: 0.11.11

2
python/ql/lib/qlpack.yml
View File

@@ -1,5 +1,5 @@
name: codeql/python-all
version: 0.11.10-dev
version: 0.11.12-dev
groups: python
dbscheme: semmlecode.python.dbscheme
extractor: python

5
python/ql/lib/semmle/python/ApiGraphs.qll
View File

@@ -328,6 +328,9 @@ module API {
*/
DataFlow::Node getInducingNode() { this = Impl::MkUse(result) or this = Impl::MkDef(result) }
/** Gets the location of this node */
PY::Location getLocation() { result = this.getInducingNode().getLocation() }
/**
* Holds if this element is at the specified location.
* The location spans column `startcolumn` of line `startline` to
@@ -335,7 +338,7 @@ module API {
* For more information, see
* [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/).
*/
predicate hasLocationInfo(
deprecated predicate hasLocationInfo(
string filepath, int startline, int startcolumn, int endline, int endcolumn
) {
this.getInducingNode().hasLocationInfo(filepath, startline, startcolumn, endline, endcolumn)

2
python/ql/lib/semmle/python/dataflow/new/DataFlow.qll
View File

@@ -24,6 +24,6 @@ private import python
module DataFlow {
private import internal.DataFlowImplSpecific
private import codeql.dataflow.DataFlow
import DataFlowMake<PythonDataFlow>
import DataFlowMake<Location, PythonDataFlow>
import internal.DataFlowImpl1
}

2
python/ql/lib/semmle/python/dataflow/new/TaintTracking.qll
View File

@@ -19,6 +19,6 @@ module TaintTracking {
private import semmle.python.dataflow.new.internal.DataFlowImplSpecific
private import semmle.python.dataflow.new.internal.TaintTrackingImplSpecific
private import codeql.dataflow.TaintTracking
import TaintFlowMake<PythonDataFlow, PythonTaintTracking>
import TaintFlowMake<Location, PythonDataFlow, PythonTaintTracking>
import internal.tainttracking1.TaintTrackingImpl
}

2
python/ql/lib/semmle/python/dataflow/new/internal/DataFlowDispatch.qll
View File

@@ -1611,7 +1611,7 @@ class FlowSummaryNode extends Node, TFlowSummaryNode {
override string toString() { result = this.getSummaryNode().toString() }
// Hack to return "empty location"
override predicate hasLocationInfo(
deprecated override predicate hasLocationInfo(
string file, int startline, int startcolumn, int endline, int endcolumn
) {
file = "" and

3
python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl.qll
View File

@@ -1,3 +1,4 @@
private import DataFlowImplSpecific
private import codeql.dataflow.internal.DataFlowImpl
import MakeImpl<PythonDataFlow>
private import semmle.python.Files
import MakeImpl<Location, PythonDataFlow>

2
python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl1.qll
View File

@@ -285,6 +285,8 @@ deprecated private module Config implements FullStateConfigSig {
int fieldFlowBranchLimit() { result = min(any(Configuration config).fieldFlowBranchLimit()) }
int accessPathLimit() { result = 5 }
FlowFeature getAFeature() { result = any(Configuration config).getAFeature() }
predicate sourceGrouping(Node source, string sourceGroup) {

2
python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl2.qll
View File

@@ -285,6 +285,8 @@ deprecated private module Config implements FullStateConfigSig {
int fieldFlowBranchLimit() { result = min(any(Configuration config).fieldFlowBranchLimit()) }
int accessPathLimit() { result = 5 }
FlowFeature getAFeature() { result = any(Configuration config).getAFeature() }
predicate sourceGrouping(Node source, string sourceGroup) {

2
python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl3.qll
View File

@@ -285,6 +285,8 @@ deprecated private module Config implements FullStateConfigSig {
int fieldFlowBranchLimit() { result = min(any(Configuration config).fieldFlowBranchLimit()) }
int accessPathLimit() { result = 5 }
FlowFeature getAFeature() { result = any(Configuration config).getAFeature() }
predicate sourceGrouping(Node source, string sourceGroup) {

2
python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl4.qll
View File

@@ -285,6 +285,8 @@ deprecated private module Config implements FullStateConfigSig {
int fieldFlowBranchLimit() { result = min(any(Configuration config).fieldFlowBranchLimit()) }
int accessPathLimit() { result = 5 }
FlowFeature getAFeature() { result = any(Configuration config).getAFeature() }
predicate sourceGrouping(Node source, string sourceGroup) {

3
python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImplCommon.qll
View File

@@ -1,3 +1,4 @@
private import DataFlowImplSpecific
private import codeql.dataflow.internal.DataFlowImplCommon
import MakeImplCommon<PythonDataFlow>
private import semmle.python.Files
import MakeImplCommon<Location, PythonDataFlow>

4
python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImplSpecific.qll
View File

@@ -15,11 +15,13 @@ module Public {
import DataFlowUtil
}
module PythonDataFlow implements InputSig {
module PythonDataFlow implements InputSig<Python::Location> {
import Private
import Public
predicate neverSkipInPathGraph = Private::neverSkipInPathGraph/1;
Node exprNode(DataFlowExpr e) { result = Public::exprNode(e) }
predicate ignoreFieldFlowBranchLimit(DataFlowCallable c) { exists(c.asLibraryCallable()) }
}

4
python/ql/lib/semmle/python/dataflow/new/internal/DataFlowPublic.qll
View File

@@ -148,6 +148,7 @@ class Node extends TNode {
DataFlowCallable getEnclosingCallable() { result = getCallableScope(this.getScope()) }
/** Gets the location of this node */
cached
Location getLocation() { none() }
/**
@@ -157,8 +158,7 @@ class Node extends TNode {
* For more information, see
* [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/).
*/
cached
predicate hasLocationInfo(
deprecated predicate hasLocationInfo(
string filepath, int startline, int startcolumn, int endline, int endcolumn
) {
Stages::DataFlow::ref() and

4
python/ql/lib/semmle/python/dataflow/new/internal/FlowSummaryImpl.qll
View File

@@ -9,7 +9,7 @@ private import DataFlowImplSpecific as DataFlowImplSpecific
private import DataFlowImplSpecific::Private
private import DataFlowImplSpecific::Public
module Input implements InputSig<DataFlowImplSpecific::PythonDataFlow> {
module Input implements InputSig<Location, DataFlowImplSpecific::PythonDataFlow> {
class SummarizedCallableBase = string;
ArgumentPosition callbackSelfParameterPosition() { result.isLambdaSelf() }
@@ -88,7 +88,7 @@ module Input implements InputSig<DataFlowImplSpecific::PythonDataFlow> {
}
}
private import Make<DataFlowImplSpecific::PythonDataFlow, Input> as Impl
private import Make<Location, DataFlowImplSpecific::PythonDataFlow, Input> as Impl
private module StepsInput implements Impl::Private::StepsInputSig {
DataFlowCall getACall(Public::SummarizedCallable sc) {

3
python/ql/lib/semmle/python/dataflow/new/internal/TaintTrackingImplSpecific.qll
View File

@@ -4,7 +4,8 @@
private import codeql.dataflow.TaintTracking
private import DataFlowImplSpecific
private import semmle.python.Files
module PythonTaintTracking implements InputSig<PythonDataFlow> {
module PythonTaintTracking implements InputSig<Location, PythonDataFlow> {
import TaintTrackingPrivate
}

2
python/ql/lib/semmle/python/internal/CachedStages.qll
View File

@@ -194,7 +194,7 @@ module Stages {
or
exists(any(DataFlowPublic::Node node).toString())
or
any(DataFlowPublic::Node node).hasLocationInfo(_, _, _, _, _)
exists(any(DataFlowPublic::Node node).getLocation())
or
DataFlowDispatch::resolveCall(_, _, _)
or

10
python/ql/src/CHANGELOG.md
View File

@@ -1,3 +1,13 @@
## 0.9.11
No user-facing changes.
## 0.9.10
### New Queries
* The query `py/nosql-injection` for finding NoSQL injection vulnerabilities is now part of the default security suite.
## 0.9.9
No user-facing changes.

16
python/ql/src/Security/CWE-798/HardcodedCredentials.ql
View File

@@ -16,6 +16,8 @@ import python
import semmle.python.dataflow.new.DataFlow
import semmle.python.dataflow.new.TaintTracking
import semmle.python.filters.Tests
private import semmle.python.dataflow.new.internal.DataFlowDispatch as DataFlowDispatch
private import semmle.python.dataflow.new.internal.Builtins::Builtins as Builtins
bindingset[char, fraction]
predicate fewer_characters_than(StrConst str, string char, float fraction) {
@@ -30,15 +32,13 @@ predicate fewer_characters_than(StrConst str, string char, float fraction) {
}
predicate possible_reflective_name(string name) {
exists(any(ModuleValue m).attr(name))
any(Function f).getName() = name
or
exists(any(ClassValue c).lookup(name))
any(Class c).getName() = name
or
any(ClassValue c).getName() = name
any(Module m).getName() = name
or
exists(Module::named(name))
or
exists(Value::named(name))
exists(Builtins::likelyBuiltin(name))
}
int char_count(StrConst str) { result = count(string c | c = str.getText().charAt(_)) }
@@ -84,7 +84,9 @@ class CredentialSink extends DataFlow::Node {
name.regexpMatch(getACredentialRegex()) and
not name.matches("%file")
|
any(FunctionValue func).getNamedArgumentForCall(_, name) = this.asCfgNode()
exists(DataFlowDispatch::ArgumentPosition pos | pos.isKeyword(name) |
this.(DataFlow::ArgumentNode).argumentOf(_, pos)
)
or
exists(Keyword k | k.getArg() = name and k.getValue().getAFlowNode() = this.asCfgNode())
or

1
python/ql/src/Summary/LinesOfUserCode.ql
View File

@@ -8,6 +8,7 @@
* @kind metric
* @tags summary
* lines-of-code
* debug
* @id py/summary/lines-of-user-code
*/

7
python/ql/src/change-notes/2024-03-04-nosql-injection.md → python/ql/src/change-notes/released/0.9.10.md
View File

@@ -1,4 +1,5 @@
---
category: newQuery
---
## 0.9.10
### New Queries
* The query `py/nosql-injection` for finding NoSQL injection vulnerabilities is now part of the default security suite.

3
python/ql/src/change-notes/released/0.9.11.md Normal file
View File

@@ -0,0 +1,3 @@
## 0.9.11
No user-facing changes.

2
python/ql/src/codeql-pack.release.yml
View File

@@ -1,2 +1,2 @@
---
lastReleaseVersion: 0.9.9
lastReleaseVersion: 0.9.11

38
python/ql/src/experimental/Security/CWE-770/UnicodeDoS.qhelp Normal file
View File

@@ -0,0 +1,38 @@
<!DOCTYPE qhelp PUBLIC
"-//Semmle//qhelp//EN"
"qhelp.dtd">
<qhelp>
<overview>
<p>When a remote user-controlled data can reach a costly Unicode normalization with either form, NFKC or NFKD, an attack such as the One Million Unicode Characters, could lead to a denial of service on Windows OS.</p>
<p>And, with the use of special Unicode characters, like U+2100 (℀) or U+2105 (℅), the payload size could be tripled after the compatibility normalization.</p>
</overview>
<recommendation>
<p>Ensure limiting the size of any incoming data that would go through a costly operations, including a Windows Unicode normalization with NFKC or NFKD. Such a recommandation would avoid a potential denial of service.</p>
</recommendation>
<example>
<p>
In this example a simple user-controlled data reaches a Unicode normalization with the form "NFKC".
</p>
<sample src="bad.py" />
<p>To fix this vulnerability, we need restrain the size of the user input.</p>
<p>For example, we can use the <code>len()</code> builtin function to limit the size of the user input.</p>
<sample src="good.py" />
</example>
<references>
<li>
<a href="https://hackerone.com/reports/2258758">CVE-2023-46695: Potential denial of service vulnerability in Django UsernameField on Windows.</a>
</li>
</references>
</qhelp>

114
python/ql/src/experimental/Security/CWE-770/UnicodeDoS.ql Normal file
View File

@@ -0,0 +1,114 @@
/**
* @name Denial of Service using Unicode Characters
* @description A remote user-controlled data can reach a costly Unicode normalization with either form NFKC or NFKD. On Windows OS, with an attack such as the One Million Unicode Characters, this could lead to a denial of service. And, with the use of special Unicode characters, like U+2100 (℀) or U+2105 (℅), the payload size could be tripled.
* @kind path-problem
* @id py/unicode-dos
* @precision high
* @problem.severity error
* @tags security
* experimental
* external/cwe/cwe-770
*/
import python
import semmle.python.ApiGraphs
import semmle.python.Concepts
import semmle.python.dataflow.new.TaintTracking
import semmle.python.dataflow.new.internal.DataFlowPublic
import semmle.python.dataflow.new.RemoteFlowSources
// The Unicode compatibility normalization calls from unicodedata, unidecode, pyunormalize
// and textnorm modules. The use of argIdx is to constraint the argument being normalized.
class UnicodeCompatibilityNormalize extends API::CallNode {
int argIdx;
UnicodeCompatibilityNormalize() {
(
this = API::moduleImport("unicodedata").getMember("normalize").getACall() and
this.getParameter(0).getAValueReachingSink().asExpr().(StrConst).getText() in ["NFKC", "NFKD"]
or
this = API::moduleImport("pyunormalize").getMember("normalize").getACall() and
this.getParameter(0).getAValueReachingSink().asExpr().(StrConst).getText() in ["NFKC", "NFKD"]
) and
argIdx = 1
or
(
this = API::moduleImport("textnorm").getMember("normalize_unicode").getACall() and
this.getParameter(1).getAValueReachingSink().asExpr().(StrConst).getText() in ["NFKC", "NFKD"]
or
this = API::moduleImport("unidecode").getMember("unidecode").getACall()
or
this = API::moduleImport("pyunormalize").getMember(["NFKC", "NFKD"]).getACall()
) and
argIdx = 0
}
DataFlow::Node getPathArg() { result = this.getArg(argIdx) }
}
predicate underAValue(DataFlow::GuardNode g, ControlFlowNode node, boolean branch) {
exists(CompareNode cn | cn = g |
exists(API::CallNode lenCall, Cmpop op, Node n |
lenCall = n.getALocalSource() and
(
// arg <= LIMIT OR arg < LIMIT
(op instanceof LtE or op instanceof Lt) and
branch = true and
cn.operands(n.asCfgNode(), op, _)
or
// LIMIT >= arg OR LIMIT > arg
(op instanceof GtE or op instanceof Gt) and
branch = true and
cn.operands(_, op, n.asCfgNode())
or
// not arg >= LIMIT OR not arg > LIMIT
(op instanceof GtE or op instanceof Gt) and
branch = false and
cn.operands(n.asCfgNode(), op, _)
or
// not LIMIT <= arg OR not LIMIT < arg
(op instanceof LtE or op instanceof Lt) and
branch = false and
cn.operands(_, op, n.asCfgNode())
)
|
lenCall = API::builtin("len").getACall() and
node = lenCall.getArg(0).asCfgNode()
) //and
//not cn.getLocation().getFile().inStdlib()
)
}
private module UnicodeDoSConfig implements DataFlow::ConfigSig {
predicate isSource(DataFlow::Node source) { source instanceof RemoteFlowSource }
predicate isBarrier(DataFlow::Node sanitizer) {
// underAValue is a check to ensure that the length of the user-provided value is limited to a certain amount
sanitizer = DataFlow::BarrierGuard<underAValue/3>::getABarrierNode()
}
predicate isSink(DataFlow::Node sink) {
// Any call to the Unicode compatibility normalization is a costly operation
sink = any(UnicodeCompatibilityNormalize ucn).getPathArg()
or
// The call to secure_filename() from pallets/werkzeug uses the Unicode compatibility normalization
// under the hood, https://github.com/pallets/werkzeug/blob/d3dd65a27388fbd39d146caacf2563639ba622f0/src/werkzeug/utils.py#L218
sink = API::moduleImport("werkzeug").getMember("secure_filename").getACall().getArg(_)
or
sink =
API::moduleImport("werkzeug")
.getMember("utils")
.getMember("secure_filename")
.getACall()
.getArg(_)
}
}
module UnicodeDoSFlow = TaintTracking::Global<UnicodeDoSConfig>;
import UnicodeDoSFlow::PathGraph
from UnicodeDoSFlow::PathNode source, UnicodeDoSFlow::PathNode sink
where UnicodeDoSFlow::flowPath(source, sink)
select sink.getNode(), source, sink, "This $@ can reach a $@.", source.getNode(),
"user-provided value", sink.getNode(), "costly Unicode normalization operation"

17
python/ql/src/experimental/Security/CWE-770/bad.py Normal file
View File

@@ -0,0 +1,17 @@
from flask import Flask, jsonify, request
import unicodedata
app = Flask(__name__)
@app.route("/bad_1")
def bad_1():
# User controlled data
file_path = request.args.get("file_path", "")
# Normalize the file path using NFKC Unicode normalization
return (
unicodedata.normalize("NFKC", file_path),
200,
{"Content-Type": "application/octet-stream"},
)

16
python/ql/src/experimental/Security/CWE-770/good.py Normal file
View File

@@ -0,0 +1,16 @@
from flask import Flask, jsonify, request
import unicodedata
app = Flask(__name__)
@app.route("/good_1")
def good_1():
r = request.args.get("file_path", "")
if len(r) <= 1_000:
# Normalize the r using NFKD Unicode normalization
r = unicodedata.normalize("NFKD", r)
return r, 200, {"Content-Type": "application/octet-stream"}
else:
return jsonify({"error": "File not found"}), 404

2
python/ql/src/qlpack.yml
View File

@@ -1,5 +1,5 @@
name: codeql/python-queries
version: 0.9.10-dev
version: 0.9.12-dev
groups:
- python
- queries

4
python/ql/test/experimental/dataflow/summaries/InlineTaintTest.expected Normal file
View File

@@ -0,0 +1,4 @@
argumentToEnsureNotTaintedNotMarkedAsSpurious
untaintedArgumentToEnsureTaintedNotMarkedAsMissing
testFailures
failures

4
python/ql/test/experimental/dataflow/summaries/InlineTaintTest.ql Normal file
View File

@@ -0,0 +1,4 @@
import python
import experimental.meta.InlineTaintTest
import MakeInlineTaintTest<TestTaintTrackingConfig>
import TestSummaries

105
python/ql/test/experimental/dataflow/summaries/TestSummaries.qll
View File

@@ -136,3 +136,108 @@ private class SummarizedCallableJsonLoads extends SummarizedCallable {
preservesValue = true
}
}
// Repeated summaries
private class SummarizedCallableWithSubpath extends SummarizedCallable {
SummarizedCallableWithSubpath() { this = "extracted_package.functions.with_subpath" }
override DataFlow::CallCfgNode getACall() {
result =
API::moduleImport("extracted_package")
.getMember("functions")
.getMember("with_subpath")
.getACall()
}
override DataFlow::ArgumentNode getACallback() {
result =
API::moduleImport("extracted_package")
.getMember("functions")
.getMember("with_subpath")
.getAValueReachableFromSource()
}
override predicate propagatesFlow(string input, string output, boolean preservesValue) {
input = "Argument[0]" and
output = "ReturnValue" and
preservesValue = false
}
}
private class SummarizedCallableWithSubpathAgain extends SummarizedCallable {
SummarizedCallableWithSubpathAgain() { this = "extracted_package.functions.with_subpathII" }
override DataFlow::CallCfgNode getACall() {
result =
API::moduleImport("extracted_package")
.getMember("functions")
.getMember("with_subpath")
.getACall()
}
override DataFlow::ArgumentNode getACallback() {
result =
API::moduleImport("extracted_package")
.getMember("functions")
.getMember("with_subpath")
.getAValueReachableFromSource()
}
override predicate propagatesFlow(string input, string output, boolean preservesValue) {
input = "Argument[0]" and
output = "ReturnValue.Attribute[pattern]" and
preservesValue = true
}
}
private class SummarizedCallableWithoutSubpath extends SummarizedCallable {
SummarizedCallableWithoutSubpath() { this = "extracted_package.functions.without_subpath" }
override DataFlow::CallCfgNode getACall() {
result =
API::moduleImport("extracted_package")
.getMember("functions")
.getMember("without_subpath")
.getACall()
}
override DataFlow::ArgumentNode getACallback() {
result =
API::moduleImport("extracted_package")
.getMember("functions")
.getMember("without_subpath")
.getAValueReachableFromSource()
}
override predicate propagatesFlow(string input, string output, boolean preservesValue) {
input = "Argument[0]" and
output = "ReturnValue" and
preservesValue = false
}
}
private class SummarizedCallableWithoutSubpathAgain extends SummarizedCallable {
SummarizedCallableWithoutSubpathAgain() { this = "extracted_package.functions.without_subpathII" }
override DataFlow::CallCfgNode getACall() {
result =
API::moduleImport("extracted_package")
.getMember("functions")
.getMember("without_subpath")
.getACall()
}
override DataFlow::ArgumentNode getACallback() {
result =
API::moduleImport("extracted_package")
.getMember("functions")
.getMember("without_subpath")
.getAValueReachableFromSource()
}
override predicate propagatesFlow(string input, string output, boolean preservesValue) {
input = "Argument[0]" and
output = "ReturnValue.Attribute[pattern]" and
preservesValue = true
}
}

18
python/ql/test/experimental/dataflow/summaries/conflicting_summaries.py Normal file
View File

@@ -0,0 +1,18 @@
# Bad interaction of two summaries for the same function
ts = TAINTED_STRING
from extracted_package.functions import with_subpath, without_subpath
# For the function `with_subpath`, flow from the first argument to the return value
# can be concluded from its definition. This seems to discard all summaries, including
# the one with flow to `ReturnValue.Attribute[pattern]`.
ensure_tainted(
with_subpath(ts).pattern, # $ tainted
with_subpath(ts), # $ tainted
with_subpath(ts), # $ tainted
)
ensure_tainted(
without_subpath(ts).pattern, # $ tainted
without_subpath(ts), # $ tainted
without_subpath(ts), # $ tainted
)

0
python/ql/test/experimental/dataflow/summaries/extracted_package/__init__.py Normal file
View File

5
python/ql/test/experimental/dataflow/summaries/extracted_package/functions.py Normal file
View File

@@ -0,0 +1,5 @@
def with_subpath(x):
return x
def without_subpath(x):
pass

15
python/ql/test/experimental/dataflow/summaries/summaries.expected
View File

@@ -4,7 +4,9 @@ edges
| summaries.py:32:20:32:25 | ControlFlowNode for SOURCE | summaries.py:32:11:32:26 | ControlFlowNode for identity() | provenance | |
| summaries.py:36:1:36:14 | ControlFlowNode for tainted_lambda | summaries.py:37:6:37:19 | ControlFlowNode for tainted_lambda | provenance | |
| summaries.py:36:18:36:54 | ControlFlowNode for apply_lambda() | summaries.py:36:1:36:14 | ControlFlowNode for tainted_lambda | provenance | |
| summaries.py:36:38:36:38 | ControlFlowNode for x | summaries.py:36:41:36:45 | ControlFlowNode for BinaryExpr | provenance | |
| summaries.py:36:48:36:53 | ControlFlowNode for SOURCE | summaries.py:36:18:36:54 | ControlFlowNode for apply_lambda() | provenance | |
| summaries.py:36:48:36:53 | ControlFlowNode for SOURCE | summaries.py:36:38:36:38 | ControlFlowNode for x | provenance | |
| summaries.py:44:1:44:12 | ControlFlowNode for tainted_list | summaries.py:45:6:45:20 | ControlFlowNode for Subscript | provenance | |
| summaries.py:44:1:44:12 | ControlFlowNode for tainted_list [List element] | summaries.py:45:6:45:17 | ControlFlowNode for tainted_list [List element] | provenance | |
| summaries.py:44:16:44:33 | ControlFlowNode for reversed() | summaries.py:44:1:44:12 | ControlFlowNode for tainted_list | provenance | |
@@ -14,13 +16,17 @@ edges
| summaries.py:44:26:44:31 | ControlFlowNode for SOURCE | summaries.py:44:25:44:32 | ControlFlowNode for List | provenance | |
| summaries.py:44:26:44:31 | ControlFlowNode for SOURCE | summaries.py:44:25:44:32 | ControlFlowNode for List [List element] | provenance | |
| summaries.py:45:6:45:17 | ControlFlowNode for tainted_list [List element] | summaries.py:45:6:45:20 | ControlFlowNode for Subscript | provenance | |
| summaries.py:48:15:48:15 | ControlFlowNode for x | summaries.py:49:12:49:18 | ControlFlowNode for BinaryExpr | provenance | |
| summaries.py:51:1:51:14 | ControlFlowNode for tainted_mapped [List element] | summaries.py:52:6:52:19 | ControlFlowNode for tainted_mapped [List element] | provenance | |
| summaries.py:51:18:51:46 | ControlFlowNode for list_map() [List element] | summaries.py:51:1:51:14 | ControlFlowNode for tainted_mapped [List element] | provenance | |
| summaries.py:51:38:51:45 | ControlFlowNode for List [List element] | summaries.py:48:15:48:15 | ControlFlowNode for x | provenance | |
| summaries.py:51:38:51:45 | ControlFlowNode for List [List element] | summaries.py:51:18:51:46 | ControlFlowNode for list_map() [List element] | provenance | |
| summaries.py:51:39:51:44 | ControlFlowNode for SOURCE | summaries.py:51:38:51:45 | ControlFlowNode for List [List element] | provenance | |
| summaries.py:52:6:52:19 | ControlFlowNode for tainted_mapped [List element] | summaries.py:52:6:52:22 | ControlFlowNode for Subscript | provenance | |
| summaries.py:54:23:54:23 | ControlFlowNode for x | summaries.py:55:12:55:12 | ControlFlowNode for x | provenance | |
| summaries.py:57:1:57:23 | ControlFlowNode for tainted_mapped_explicit [List element] | summaries.py:58:6:58:28 | ControlFlowNode for tainted_mapped_explicit [List element] | provenance | |
| summaries.py:57:27:57:63 | ControlFlowNode for list_map() [List element] | summaries.py:57:1:57:23 | ControlFlowNode for tainted_mapped_explicit [List element] | provenance | |
| summaries.py:57:55:57:62 | ControlFlowNode for List [List element] | summaries.py:54:23:54:23 | ControlFlowNode for x | provenance | |
| summaries.py:57:55:57:62 | ControlFlowNode for List [List element] | summaries.py:57:27:57:63 | ControlFlowNode for list_map() [List element] | provenance | |
| summaries.py:57:56:57:61 | ControlFlowNode for SOURCE | summaries.py:57:55:57:62 | ControlFlowNode for List [List element] | provenance | |
| summaries.py:58:6:58:28 | ControlFlowNode for tainted_mapped_explicit [List element] | summaries.py:58:6:58:31 | ControlFlowNode for Subscript | provenance | |
@@ -46,6 +52,8 @@ nodes
| summaries.py:33:6:33:12 | ControlFlowNode for tainted | semmle.label | ControlFlowNode for tainted |
| summaries.py:36:1:36:14 | ControlFlowNode for tainted_lambda | semmle.label | ControlFlowNode for tainted_lambda |
| summaries.py:36:18:36:54 | ControlFlowNode for apply_lambda() | semmle.label | ControlFlowNode for apply_lambda() |
| summaries.py:36:38:36:38 | ControlFlowNode for x | semmle.label | ControlFlowNode for x |
| summaries.py:36:41:36:45 | ControlFlowNode for BinaryExpr | semmle.label | ControlFlowNode for BinaryExpr |
| summaries.py:36:48:36:53 | ControlFlowNode for SOURCE | semmle.label | ControlFlowNode for SOURCE |
| summaries.py:37:6:37:19 | ControlFlowNode for tainted_lambda | semmle.label | ControlFlowNode for tainted_lambda |
| summaries.py:44:1:44:12 | ControlFlowNode for tainted_list | semmle.label | ControlFlowNode for tainted_list |
@@ -57,12 +65,16 @@ nodes
| summaries.py:44:26:44:31 | ControlFlowNode for SOURCE | semmle.label | ControlFlowNode for SOURCE |
| summaries.py:45:6:45:17 | ControlFlowNode for tainted_list [List element] | semmle.label | ControlFlowNode for tainted_list [List element] |
| summaries.py:45:6:45:20 | ControlFlowNode for Subscript | semmle.label | ControlFlowNode for Subscript |
| summaries.py:48:15:48:15 | ControlFlowNode for x | semmle.label | ControlFlowNode for x |
| summaries.py:49:12:49:18 | ControlFlowNode for BinaryExpr | semmle.label | ControlFlowNode for BinaryExpr |
| summaries.py:51:1:51:14 | ControlFlowNode for tainted_mapped [List element] | semmle.label | ControlFlowNode for tainted_mapped [List element] |
| summaries.py:51:18:51:46 | ControlFlowNode for list_map() [List element] | semmle.label | ControlFlowNode for list_map() [List element] |
| summaries.py:51:38:51:45 | ControlFlowNode for List [List element] | semmle.label | ControlFlowNode for List [List element] |
| summaries.py:51:39:51:44 | ControlFlowNode for SOURCE | semmle.label | ControlFlowNode for SOURCE |
| summaries.py:52:6:52:19 | ControlFlowNode for tainted_mapped [List element] | semmle.label | ControlFlowNode for tainted_mapped [List element] |
| summaries.py:52:6:52:22 | ControlFlowNode for Subscript | semmle.label | ControlFlowNode for Subscript |
| summaries.py:54:23:54:23 | ControlFlowNode for x | semmle.label | ControlFlowNode for x |
| summaries.py:55:12:55:12 | ControlFlowNode for x | semmle.label | ControlFlowNode for x |
| summaries.py:57:1:57:23 | ControlFlowNode for tainted_mapped_explicit [List element] | semmle.label | ControlFlowNode for tainted_mapped_explicit [List element] |
| summaries.py:57:27:57:63 | ControlFlowNode for list_map() [List element] | semmle.label | ControlFlowNode for list_map() [List element] |
| summaries.py:57:55:57:62 | ControlFlowNode for List [List element] | semmle.label | ControlFlowNode for List [List element] |
@@ -87,6 +99,9 @@ nodes
| summaries.py:68:6:68:23 | ControlFlowNode for tainted_resultlist [List element] | semmle.label | ControlFlowNode for tainted_resultlist [List element] |
| summaries.py:68:6:68:26 | ControlFlowNode for Subscript | semmle.label | ControlFlowNode for Subscript |
subpaths
| summaries.py:36:48:36:53 | ControlFlowNode for SOURCE | summaries.py:36:38:36:38 | ControlFlowNode for x | summaries.py:36:41:36:45 | ControlFlowNode for BinaryExpr | summaries.py:36:18:36:54 | ControlFlowNode for apply_lambda() |
| summaries.py:51:38:51:45 | ControlFlowNode for List [List element] | summaries.py:48:15:48:15 | ControlFlowNode for x | summaries.py:49:12:49:18 | ControlFlowNode for BinaryExpr | summaries.py:51:18:51:46 | ControlFlowNode for list_map() [List element] |
| summaries.py:57:55:57:62 | ControlFlowNode for List [List element] | summaries.py:54:23:54:23 | ControlFlowNode for x | summaries.py:55:12:55:12 | ControlFlowNode for x | summaries.py:57:27:57:63 | ControlFlowNode for list_map() [List element] |
invalidSpecComponent
#select
| summaries.py:33:6:33:12 | ControlFlowNode for tainted | summaries.py:32:20:32:25 | ControlFlowNode for SOURCE | summaries.py:33:6:33:12 | ControlFlowNode for tainted | $@ | summaries.py:32:20:32:25 | ControlFlowNode for SOURCE | ControlFlowNode for SOURCE |

2
python/ql/test/experimental/dataflow/variable-capture/by_value.py
View File

@@ -34,7 +34,7 @@ def by_value1():
a = SOURCE
def inner(a_val=a):
SINK(a_val) #$ captured
SINK_F(a) #$ SPURIOUS: captured
SINK_F(a)
a = NONSOURCE
inner()

73
python/ql/test/experimental/query-tests/Security/CWE-770/UnicodeDoS.expected Normal file
View File

@@ -0,0 +1,73 @@
edges
| tests.py:1:35:1:41 | ControlFlowNode for ImportMember | tests.py:1:35:1:41 | ControlFlowNode for request | provenance | |
| tests.py:1:35:1:41 | ControlFlowNode for request | tests.py:12:17:12:23 | ControlFlowNode for request | provenance | |
| tests.py:1:35:1:41 | ControlFlowNode for request | tests.py:24:9:24:15 | ControlFlowNode for request | provenance | |
| tests.py:1:35:1:41 | ControlFlowNode for request | tests.py:36:9:36:15 | ControlFlowNode for request | provenance | |
| tests.py:1:35:1:41 | ControlFlowNode for request | tests.py:48:9:48:15 | ControlFlowNode for request | provenance | |
| tests.py:1:35:1:41 | ControlFlowNode for request | tests.py:60:9:60:15 | ControlFlowNode for request | provenance | |
| tests.py:1:35:1:41 | ControlFlowNode for request | tests.py:72:9:72:15 | ControlFlowNode for request | provenance | |
| tests.py:12:5:12:13 | ControlFlowNode for file_path | tests.py:16:39:16:47 | ControlFlowNode for file_path | provenance | |
| tests.py:12:17:12:23 | ControlFlowNode for request | tests.py:12:17:12:28 | ControlFlowNode for Attribute | provenance | |
| tests.py:12:17:12:28 | ControlFlowNode for Attribute | tests.py:12:17:12:49 | ControlFlowNode for Attribute() | provenance | |
| tests.py:12:17:12:49 | ControlFlowNode for Attribute() | tests.py:12:5:12:13 | ControlFlowNode for file_path | provenance | |
| tests.py:24:5:24:5 | ControlFlowNode for r | tests.py:28:43:28:43 | ControlFlowNode for r | provenance | |
| tests.py:24:9:24:15 | ControlFlowNode for request | tests.py:24:9:24:20 | ControlFlowNode for Attribute | provenance | |
| tests.py:24:9:24:20 | ControlFlowNode for Attribute | tests.py:24:9:24:33 | ControlFlowNode for Attribute() | provenance | |
| tests.py:24:9:24:33 | ControlFlowNode for Attribute() | tests.py:24:5:24:5 | ControlFlowNode for r | provenance | |
| tests.py:36:5:36:5 | ControlFlowNode for r | tests.py:40:43:40:43 | ControlFlowNode for r | provenance | |
| tests.py:36:9:36:15 | ControlFlowNode for request | tests.py:36:9:36:20 | ControlFlowNode for Attribute | provenance | |
| tests.py:36:9:36:20 | ControlFlowNode for Attribute | tests.py:36:9:36:33 | ControlFlowNode for Attribute() | provenance | |
| tests.py:36:9:36:33 | ControlFlowNode for Attribute() | tests.py:36:5:36:5 | ControlFlowNode for r | provenance | |
| tests.py:48:5:48:5 | ControlFlowNode for r | tests.py:52:43:52:43 | ControlFlowNode for r | provenance | |
| tests.py:48:9:48:15 | ControlFlowNode for request | tests.py:48:9:48:20 | ControlFlowNode for Attribute | provenance | |
| tests.py:48:9:48:20 | ControlFlowNode for Attribute | tests.py:48:9:48:33 | ControlFlowNode for Attribute() | provenance | |
| tests.py:48:9:48:33 | ControlFlowNode for Attribute() | tests.py:48:5:48:5 | ControlFlowNode for r | provenance | |
| tests.py:60:5:60:5 | ControlFlowNode for r | tests.py:64:43:64:43 | ControlFlowNode for r | provenance | |
| tests.py:60:9:60:15 | ControlFlowNode for request | tests.py:60:9:60:20 | ControlFlowNode for Attribute | provenance | |
| tests.py:60:9:60:20 | ControlFlowNode for Attribute | tests.py:60:9:60:33 | ControlFlowNode for Attribute() | provenance | |
| tests.py:60:9:60:33 | ControlFlowNode for Attribute() | tests.py:60:5:60:5 | ControlFlowNode for r | provenance | |
| tests.py:72:5:72:5 | ControlFlowNode for r | tests.py:76:43:76:43 | ControlFlowNode for r | provenance | |
| tests.py:72:9:72:15 | ControlFlowNode for request | tests.py:72:9:72:20 | ControlFlowNode for Attribute | provenance | |
| tests.py:72:9:72:20 | ControlFlowNode for Attribute | tests.py:72:9:72:33 | ControlFlowNode for Attribute() | provenance | |
| tests.py:72:9:72:33 | ControlFlowNode for Attribute() | tests.py:72:5:72:5 | ControlFlowNode for r | provenance | |
nodes
| tests.py:1:35:1:41 | ControlFlowNode for ImportMember | semmle.label | ControlFlowNode for ImportMember |
| tests.py:1:35:1:41 | ControlFlowNode for request | semmle.label | ControlFlowNode for request |
| tests.py:12:5:12:13 | ControlFlowNode for file_path | semmle.label | ControlFlowNode for file_path |
| tests.py:12:17:12:23 | ControlFlowNode for request | semmle.label | ControlFlowNode for request |
| tests.py:12:17:12:28 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute |
| tests.py:12:17:12:49 | ControlFlowNode for Attribute() | semmle.label | ControlFlowNode for Attribute() |
| tests.py:16:39:16:47 | ControlFlowNode for file_path | semmle.label | ControlFlowNode for file_path |
| tests.py:24:5:24:5 | ControlFlowNode for r | semmle.label | ControlFlowNode for r |
| tests.py:24:9:24:15 | ControlFlowNode for request | semmle.label | ControlFlowNode for request |
| tests.py:24:9:24:20 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute |
| tests.py:24:9:24:33 | ControlFlowNode for Attribute() | semmle.label | ControlFlowNode for Attribute() |
| tests.py:28:43:28:43 | ControlFlowNode for r | semmle.label | ControlFlowNode for r |
| tests.py:36:5:36:5 | ControlFlowNode for r | semmle.label | ControlFlowNode for r |
| tests.py:36:9:36:15 | ControlFlowNode for request | semmle.label | ControlFlowNode for request |
| tests.py:36:9:36:20 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute |
| tests.py:36:9:36:33 | ControlFlowNode for Attribute() | semmle.label | ControlFlowNode for Attribute() |
| tests.py:40:43:40:43 | ControlFlowNode for r | semmle.label | ControlFlowNode for r |
| tests.py:48:5:48:5 | ControlFlowNode for r | semmle.label | ControlFlowNode for r |
| tests.py:48:9:48:15 | ControlFlowNode for request | semmle.label | ControlFlowNode for request |
| tests.py:48:9:48:20 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute |
| tests.py:48:9:48:33 | ControlFlowNode for Attribute() | semmle.label | ControlFlowNode for Attribute() |
| tests.py:52:43:52:43 | ControlFlowNode for r | semmle.label | ControlFlowNode for r |
| tests.py:60:5:60:5 | ControlFlowNode for r | semmle.label | ControlFlowNode for r |
| tests.py:60:9:60:15 | ControlFlowNode for request | semmle.label | ControlFlowNode for request |
| tests.py:60:9:60:20 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute |
| tests.py:60:9:60:33 | ControlFlowNode for Attribute() | semmle.label | ControlFlowNode for Attribute() |
| tests.py:64:43:64:43 | ControlFlowNode for r | semmle.label | ControlFlowNode for r |
| tests.py:72:5:72:5 | ControlFlowNode for r | semmle.label | ControlFlowNode for r |
| tests.py:72:9:72:15 | ControlFlowNode for request | semmle.label | ControlFlowNode for request |
| tests.py:72:9:72:20 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute |
| tests.py:72:9:72:33 | ControlFlowNode for Attribute() | semmle.label | ControlFlowNode for Attribute() |
| tests.py:76:43:76:43 | ControlFlowNode for r | semmle.label | ControlFlowNode for r |
subpaths
#select
| tests.py:16:39:16:47 | ControlFlowNode for file_path | tests.py:1:35:1:41 | ControlFlowNode for ImportMember | tests.py:16:39:16:47 | ControlFlowNode for file_path | This $@ can reach a $@. | tests.py:1:35:1:41 | ControlFlowNode for ImportMember | user-provided value | tests.py:16:39:16:47 | ControlFlowNode for file_path | costly Unicode normalization operation |
| tests.py:28:43:28:43 | ControlFlowNode for r | tests.py:1:35:1:41 | ControlFlowNode for ImportMember | tests.py:28:43:28:43 | ControlFlowNode for r | This $@ can reach a $@. | tests.py:1:35:1:41 | ControlFlowNode for ImportMember | user-provided value | tests.py:28:43:28:43 | ControlFlowNode for r | costly Unicode normalization operation |
| tests.py:40:43:40:43 | ControlFlowNode for r | tests.py:1:35:1:41 | ControlFlowNode for ImportMember | tests.py:40:43:40:43 | ControlFlowNode for r | This $@ can reach a $@. | tests.py:1:35:1:41 | ControlFlowNode for ImportMember | user-provided value | tests.py:40:43:40:43 | ControlFlowNode for r | costly Unicode normalization operation |
| tests.py:52:43:52:43 | ControlFlowNode for r | tests.py:1:35:1:41 | ControlFlowNode for ImportMember | tests.py:52:43:52:43 | ControlFlowNode for r | This $@ can reach a $@. | tests.py:1:35:1:41 | ControlFlowNode for ImportMember | user-provided value | tests.py:52:43:52:43 | ControlFlowNode for r | costly Unicode normalization operation |
| tests.py:64:43:64:43 | ControlFlowNode for r | tests.py:1:35:1:41 | ControlFlowNode for ImportMember | tests.py:64:43:64:43 | ControlFlowNode for r | This $@ can reach a $@. | tests.py:1:35:1:41 | ControlFlowNode for ImportMember | user-provided value | tests.py:64:43:64:43 | ControlFlowNode for r | costly Unicode normalization operation |
| tests.py:76:43:76:43 | ControlFlowNode for r | tests.py:1:35:1:41 | ControlFlowNode for ImportMember | tests.py:76:43:76:43 | ControlFlowNode for r | This $@ can reach a $@. | tests.py:1:35:1:41 | ControlFlowNode for ImportMember | user-provided value | tests.py:76:43:76:43 | ControlFlowNode for r | costly Unicode normalization operation |

1
python/ql/test/experimental/query-tests/Security/CWE-770/UnicodeDoS.qlref Normal file
View File

@@ -0,0 +1 @@
experimental/Security/CWE-770/UnicodeDoS.ql

129
python/ql/test/experimental/query-tests/Security/CWE-770/tests.py Normal file
View File

@@ -0,0 +1,129 @@
from flask import Flask, jsonify, request
import unicodedata
app = Flask(__name__)
STATIC_DIR = "/home/unknown/"
@app.route("/bad_1")
def bad_1():
# User controlled data
file_path = request.args.get("file_path", "")
# Normalize the file path using NFKC Unicode normalization
return (
unicodedata.normalize("NFKC", file_path),
200,
{"Content-Type": "application/octet-stream"},
)
@app.route("/bad_2")
def bad_2():
r = request.args.get("r", "")
if len(r) >= 10:
# Normalize the r using NFKD Unicode normalization
r = unicodedata.normalize("NFKD", r)
return r, 200, {"Content-Type": "application/octet-stream"}
else:
return jsonify({"error": "File not found"}), 404
@app.route("/bad_3")
def bad_3():
r = request.args.get("r", "")
length = len(r)
if length >= 1_000:
# Normalize the r using NFKD Unicode normalization
r = unicodedata.normalize("NFKD", r)
return r, 200, {"Content-Type": "application/octet-stream"}
else:
return jsonify({"error": "File not found"}), 404
@app.route("/bad_4")
def bad_4():
r = request.args.get("r", "")
length = len(r)
if 1_000 <= length:
# Normalize the r using NFKD Unicode normalization
r = unicodedata.normalize("NFKD", r)
return r, 200, {"Content-Type": "application/octet-stream"}
else:
return jsonify({"error": "File not found"}), 404
@app.route("/bad_5")
def bad_5():
r = request.args.get("r", "")
length = len(r)
if not length < 1_000:
# Normalize the r using NFKD Unicode normalization
r = unicodedata.normalize("NFKD", r)
return r, 200, {"Content-Type": "application/octet-stream"}
else:
return jsonify({"error": "File not found"}), 404
@app.route("/bad_6")
def bad_6():
r = request.args.get("r", "")
length = len(r)
if not 1_000 > length:
# Normalize the r using NFKD Unicode normalization
r = unicodedata.normalize("NFKD", r)
return r, 200, {"Content-Type": "application/octet-stream"}
else:
return jsonify({"error": "File not found"}), 404
@app.route("/good_1")
def good_1():
r = request.args.get("r", "")
if len(r) <= 1_000:
# Normalize the r using NFKD Unicode normalization
r = unicodedata.normalize("NFKD", r)
return r, 200, {"Content-Type": "application/octet-stream"}
else:
return jsonify({"error": "File not found"}), 404
@app.route("/good_2")
def good_2():
r = request.args.get("r", "")
MAX_LENGTH = 1_000
length = len(r)
if length <= MAX_LENGTH:
# Normalize the r using NFKD Unicode normalization
r = unicodedata.normalize("NFKD", r)
return r, 200, {"Content-Type": "application/octet-stream"}
else:
return jsonify({"error": "File not found"}), 404
@app.route("/good_3")
def good_3():
r = request.args.get("r", "")
MAX_LENGTH = 1_000
length = len(r)
if not length >= MAX_LENGTH:
# Normalize the r using NFKD Unicode normalization
r = unicodedata.normalize("NFKD", r)
return r, 200, {"Content-Type": "application/octet-stream"}
else:
return jsonify({"error": "File not found"}), 404
@app.route("/good_4")
def good_4():
r = request.args.get("r", "")
MAX_LENGTH = 1_000
length = len(r)
if not MAX_LENGTH <= length:
# Normalize the r using NFKD Unicode normalization
r = unicodedata.normalize("NFKD", r)
return r, 200, {"Content-Type": "application/octet-stream"}
else:
return jsonify({"error": "File not found"}), 404

6
python/ql/test/library-tests/essa/ssa-compute/CONSISTENCY/TypeTrackingConsistency.expected Normal file
View File

@@ -0,0 +1,6 @@
unreachableNode
| test2.py:16:17:16:17 | ControlFlowNode for y | Unreachable node in step of kind load bar. |
| test2.py:25:23:25:23 | ControlFlowNode for x | Unreachable node in step of kind load attribute. |
| test2.py:25:23:25:23 | ControlFlowNode for x | Unreachable node in step of kind simpleLocalSmallStep. |
| test2.py:26:17:26:17 | ControlFlowNode for y | Unreachable node in step of kind load bar. |
| test2.py:27:23:27:23 | ControlFlowNode for x | Unreachable node in step of kind simpleLocalSmallStep. |

11
python/tools/BUILD.bazel Normal file
View File

@@ -0,0 +1,11 @@
load("@semmle_code//:dist.bzl", "pack_zip")
pack_zip(
name = "tools",
srcs = glob(["**/*"]),
excludes = [
"BUILD.bazel",
] + glob(["recorded-call-graph-metrics/**"]),
prefix = "tools",
visibility = ["//visibility:public"],
)

8
python/tools/autobuild.cmd Normal file
View File

@@ -0,0 +1,8 @@
@echo off
rem Legacy environment variables for the autobuild infrastructure.
set LGTM_SRC=%CD%
set LGTM_WORKSPACE=%CODEQL_EXTRACTOR_PYTHON_SCRATCH_DIR%
type NUL && python "%CODEQL_EXTRACTOR_PYTHON_ROOT%\tools\index.py"
exit /b %ERRORLEVEL%

18
python/tools/autobuild.sh Executable file
View File

@@ -0,0 +1,18 @@
#!/bin/sh
set -eu
# Legacy environment variables for the autobuild infrastructure.
LGTM_SRC="$(pwd)"
LGTM_WORKSPACE="$CODEQL_EXTRACTOR_PYTHON_SCRATCH_DIR"
export LGTM_SRC
export LGTM_WORKSPACE
if which python3 >/dev/null; then
exec python3 "$CODEQL_EXTRACTOR_PYTHON_ROOT/tools/index.py"
elif which python >/dev/null; then
exec python "$CODEQL_EXTRACTOR_PYTHON_ROOT/tools/index.py"
else
echo "ERROR: Could not find a valid Python distribution. It should be available when running 'which python' or 'which python3' in your shell. Python 2 is no longer supported."
exit 1
fi

3
python/tools/lgtm-scripts/index.cmd Normal file
View File

@@ -0,0 +1,3 @@
@echo off
py "%CODEQL_EXTRACTOR_PYTHON_ROOT%\tools\index.py"

5
python/tools/lgtm-scripts/index.sh Executable file
View File

@@ -0,0 +1,5 @@
#! /bin/bash
set -eu
python "${CODEQL_EXTRACTOR_PYTHON_ROOT}/tools/index.py"

3
python/tools/lgtm-scripts/python_setup.cmd Normal file
View File

@@ -0,0 +1,3 @@
@echo off
py "%CODEQL_EXTRACTOR_PYTHON_ROOT%\tools\setup.py" || EXIT /B 0

5
python/tools/lgtm-scripts/python_setup.sh Executable file
View File

@@ -0,0 +1,5 @@
#! /bin/bash
set -eu
python "${CODEQL_EXTRACTOR_PYTHON_ROOT}/tools/setup.py" || true

11
python/tools/pre-finalize.cmd Normal file
View File

@@ -0,0 +1,11 @@
@echo off
type NUL && "%CODEQL_DIST%\codeql" database index-files ^
--include-extension=.yaml ^
--include-extension=.yml ^
--size-limit=5m ^
--language yaml ^
-- ^
"%CODEQL_EXTRACTOR_PYTHON_WIP_DATABASE%"
exit /b %ERRORLEVEL%

11
python/tools/pre-finalize.sh Executable file
View File

@@ -0,0 +1,11 @@
#!/bin/bash
set -eu
"$CODEQL_DIST/codeql" database index-files \
--include-extension=.yaml \
--include-extension=.yml \
--size-limit=5m \
--language yaml \
-- \
"$CODEQL_EXTRACTOR_PYTHON_WIP_DATABASE"