From e2abd3ff1373fc2f2fa0f82514d773475e0ac2ae Mon Sep 17 00:00:00 2001 From: Anders Starcke Henriksen Date: Thu, 3 Aug 2023 13:47:30 +0200 Subject: [PATCH 001/122] Create separate automodel pack. --- java/ql/automodel/qlpack.yml | 7 +++++++ .../Telemetry => automodel/src}/AutomodelAlertSinkUtil.qll | 0 .../Telemetry => automodel/src}/AutomodelAlertSinks.ql | 0 .../src}/AutomodelAlertSinksPerQuery.ql | 0 .../src}/AutomodelApplicationModeCharacteristics.qll | 0 .../src}/AutomodelApplicationModeExtractCandidates.ql | 0 .../AutomodelApplicationModeExtractNegativeExamples.ql | 0 .../AutomodelApplicationModeExtractPositiveExamples.ql | 0 .../src}/AutomodelCountGeneratedSinks.ql | 0 .../Telemetry => automodel/src}/AutomodelEndpointTypes.qll | 0 .../src}/AutomodelFrameworkModeCharacteristics.qll | 0 .../src}/AutomodelFrameworkModeExtractCandidates.ql | 0 .../src}/AutomodelFrameworkModeExtractNegativeExamples.ql | 0 .../src}/AutomodelFrameworkModeExtractPositiveExamples.ql | 0 .../{src/Telemetry => automodel/src}/AutomodelJavaUtil.qll | 0 .../src}/AutomodelSharedCharacteristics.qll | 0 .../src}/AutomodelSharedGetCallable.qll | 0 .../src}/AutomodelSinkModelMrvaQueries.ql | 0 .../AutomodelApplicationModeExtractCandidates.expected | 0 .../AutomodelApplicationModeExtractCandidates.qlref | 0 ...utomodelApplicationModeExtractNegativeExamples.expected | 0 .../AutomodelApplicationModeExtractNegativeExamples.qlref | 0 ...utomodelApplicationModeExtractPositiveExamples.expected | 0 .../AutomodelApplicationModeExtractPositiveExamples.qlref | 0 .../test}/AutomodelApplicationModeExtraction/Test.java | 0 .../AutomodelFrameworkModeExtractCandidates.expected | 0 .../AutomodelFrameworkModeExtractCandidates.qlref | 0 .../AutomodelFrameworkModeExtractNegativeExamples.expected | 0 .../AutomodelFrameworkModeExtractNegativeExamples.qlref | 0 .../AutomodelFrameworkModeExtractPositiveExamples.expected | 0 .../AutomodelFrameworkModeExtractPositiveExamples.qlref | 0 .../com/github/codeql/test/NonPublicClass.java | 0 .../com/github/codeql/test/PublicClass.java | 0 .../com/github/codeql/test/PublicInterface.java | 0 .../AutomodelFrameworkModeExtraction/java/io/File.java | 0 .../java/nio/file/Files.java | 0 36 files changed, 7 insertions(+) create mode 100644 java/ql/automodel/qlpack.yml rename java/ql/{src/Telemetry => automodel/src}/AutomodelAlertSinkUtil.qll (100%) rename java/ql/{src/Telemetry => automodel/src}/AutomodelAlertSinks.ql (100%) rename java/ql/{src/Telemetry => automodel/src}/AutomodelAlertSinksPerQuery.ql (100%) rename java/ql/{src/Telemetry => automodel/src}/AutomodelApplicationModeCharacteristics.qll (100%) rename java/ql/{src/Telemetry => automodel/src}/AutomodelApplicationModeExtractCandidates.ql (100%) rename java/ql/{src/Telemetry => automodel/src}/AutomodelApplicationModeExtractNegativeExamples.ql (100%) rename java/ql/{src/Telemetry => automodel/src}/AutomodelApplicationModeExtractPositiveExamples.ql (100%) rename java/ql/{src/Telemetry => automodel/src}/AutomodelCountGeneratedSinks.ql (100%) rename java/ql/{src/Telemetry => automodel/src}/AutomodelEndpointTypes.qll (100%) rename java/ql/{src/Telemetry => automodel/src}/AutomodelFrameworkModeCharacteristics.qll (100%) rename java/ql/{src/Telemetry => automodel/src}/AutomodelFrameworkModeExtractCandidates.ql (100%) rename java/ql/{src/Telemetry => automodel/src}/AutomodelFrameworkModeExtractNegativeExamples.ql (100%) rename java/ql/{src/Telemetry => automodel/src}/AutomodelFrameworkModeExtractPositiveExamples.ql (100%) rename java/ql/{src/Telemetry => automodel/src}/AutomodelJavaUtil.qll (100%) rename java/ql/{src/Telemetry => automodel/src}/AutomodelSharedCharacteristics.qll (100%) rename java/ql/{src/Telemetry => automodel/src}/AutomodelSharedGetCallable.qll (100%) rename java/ql/{src/Telemetry => automodel/src}/AutomodelSinkModelMrvaQueries.ql (100%) rename java/ql/{test/query-tests/Telemetry => automodel/test}/AutomodelApplicationModeExtraction/AutomodelApplicationModeExtractCandidates.expected (100%) rename java/ql/{test/query-tests/Telemetry => automodel/test}/AutomodelApplicationModeExtraction/AutomodelApplicationModeExtractCandidates.qlref (100%) rename java/ql/{test/query-tests/Telemetry => automodel/test}/AutomodelApplicationModeExtraction/AutomodelApplicationModeExtractNegativeExamples.expected (100%) rename java/ql/{test/query-tests/Telemetry => automodel/test}/AutomodelApplicationModeExtraction/AutomodelApplicationModeExtractNegativeExamples.qlref (100%) rename java/ql/{test/query-tests/Telemetry => automodel/test}/AutomodelApplicationModeExtraction/AutomodelApplicationModeExtractPositiveExamples.expected (100%) rename java/ql/{test/query-tests/Telemetry => automodel/test}/AutomodelApplicationModeExtraction/AutomodelApplicationModeExtractPositiveExamples.qlref (100%) rename java/ql/{test/query-tests/Telemetry => automodel/test}/AutomodelApplicationModeExtraction/Test.java (100%) rename java/ql/{test/query-tests/Telemetry => automodel/test}/AutomodelFrameworkModeExtraction/AutomodelFrameworkModeExtractCandidates.expected (100%) rename java/ql/{test/query-tests/Telemetry => automodel/test}/AutomodelFrameworkModeExtraction/AutomodelFrameworkModeExtractCandidates.qlref (100%) rename java/ql/{test/query-tests/Telemetry => automodel/test}/AutomodelFrameworkModeExtraction/AutomodelFrameworkModeExtractNegativeExamples.expected (100%) rename java/ql/{test/query-tests/Telemetry => automodel/test}/AutomodelFrameworkModeExtraction/AutomodelFrameworkModeExtractNegativeExamples.qlref (100%) rename java/ql/{test/query-tests/Telemetry => automodel/test}/AutomodelFrameworkModeExtraction/AutomodelFrameworkModeExtractPositiveExamples.expected (100%) rename java/ql/{test/query-tests/Telemetry => automodel/test}/AutomodelFrameworkModeExtraction/AutomodelFrameworkModeExtractPositiveExamples.qlref (100%) rename java/ql/{test/query-tests/Telemetry => automodel/test}/AutomodelFrameworkModeExtraction/com/github/codeql/test/NonPublicClass.java (100%) rename java/ql/{test/query-tests/Telemetry => automodel/test}/AutomodelFrameworkModeExtraction/com/github/codeql/test/PublicClass.java (100%) rename java/ql/{test/query-tests/Telemetry => automodel/test}/AutomodelFrameworkModeExtraction/com/github/codeql/test/PublicInterface.java (100%) rename java/ql/{test/query-tests/Telemetry => automodel/test}/AutomodelFrameworkModeExtraction/java/io/File.java (100%) rename java/ql/{test/query-tests/Telemetry => automodel/test}/AutomodelFrameworkModeExtraction/java/nio/file/Files.java (100%) diff --git a/java/ql/automodel/qlpack.yml b/java/ql/automodel/qlpack.yml new file mode 100644 index 00000000000..12d756bc67d --- /dev/null +++ b/java/ql/automodel/qlpack.yml @@ -0,0 +1,7 @@ +name: codeql/java-automodel-queries +version: 0.0.1 +dependencies: + codeql/java-all: ${workspace} +extractor: java +tests: test +warnOnImplicitThis: true \ No newline at end of file diff --git a/java/ql/src/Telemetry/AutomodelAlertSinkUtil.qll b/java/ql/automodel/src/AutomodelAlertSinkUtil.qll similarity index 100% rename from java/ql/src/Telemetry/AutomodelAlertSinkUtil.qll rename to java/ql/automodel/src/AutomodelAlertSinkUtil.qll diff --git a/java/ql/src/Telemetry/AutomodelAlertSinks.ql b/java/ql/automodel/src/AutomodelAlertSinks.ql similarity index 100% rename from java/ql/src/Telemetry/AutomodelAlertSinks.ql rename to java/ql/automodel/src/AutomodelAlertSinks.ql diff --git a/java/ql/src/Telemetry/AutomodelAlertSinksPerQuery.ql b/java/ql/automodel/src/AutomodelAlertSinksPerQuery.ql similarity index 100% rename from java/ql/src/Telemetry/AutomodelAlertSinksPerQuery.ql rename to java/ql/automodel/src/AutomodelAlertSinksPerQuery.ql diff --git a/java/ql/src/Telemetry/AutomodelApplicationModeCharacteristics.qll b/java/ql/automodel/src/AutomodelApplicationModeCharacteristics.qll similarity index 100% rename from java/ql/src/Telemetry/AutomodelApplicationModeCharacteristics.qll rename to java/ql/automodel/src/AutomodelApplicationModeCharacteristics.qll diff --git a/java/ql/src/Telemetry/AutomodelApplicationModeExtractCandidates.ql b/java/ql/automodel/src/AutomodelApplicationModeExtractCandidates.ql similarity index 100% rename from java/ql/src/Telemetry/AutomodelApplicationModeExtractCandidates.ql rename to java/ql/automodel/src/AutomodelApplicationModeExtractCandidates.ql diff --git a/java/ql/src/Telemetry/AutomodelApplicationModeExtractNegativeExamples.ql b/java/ql/automodel/src/AutomodelApplicationModeExtractNegativeExamples.ql similarity index 100% rename from java/ql/src/Telemetry/AutomodelApplicationModeExtractNegativeExamples.ql rename to java/ql/automodel/src/AutomodelApplicationModeExtractNegativeExamples.ql diff --git a/java/ql/src/Telemetry/AutomodelApplicationModeExtractPositiveExamples.ql b/java/ql/automodel/src/AutomodelApplicationModeExtractPositiveExamples.ql similarity index 100% rename from java/ql/src/Telemetry/AutomodelApplicationModeExtractPositiveExamples.ql rename to java/ql/automodel/src/AutomodelApplicationModeExtractPositiveExamples.ql diff --git a/java/ql/src/Telemetry/AutomodelCountGeneratedSinks.ql b/java/ql/automodel/src/AutomodelCountGeneratedSinks.ql similarity index 100% rename from java/ql/src/Telemetry/AutomodelCountGeneratedSinks.ql rename to java/ql/automodel/src/AutomodelCountGeneratedSinks.ql diff --git a/java/ql/src/Telemetry/AutomodelEndpointTypes.qll b/java/ql/automodel/src/AutomodelEndpointTypes.qll similarity index 100% rename from java/ql/src/Telemetry/AutomodelEndpointTypes.qll rename to java/ql/automodel/src/AutomodelEndpointTypes.qll diff --git a/java/ql/src/Telemetry/AutomodelFrameworkModeCharacteristics.qll b/java/ql/automodel/src/AutomodelFrameworkModeCharacteristics.qll similarity index 100% rename from java/ql/src/Telemetry/AutomodelFrameworkModeCharacteristics.qll rename to java/ql/automodel/src/AutomodelFrameworkModeCharacteristics.qll diff --git a/java/ql/src/Telemetry/AutomodelFrameworkModeExtractCandidates.ql b/java/ql/automodel/src/AutomodelFrameworkModeExtractCandidates.ql similarity index 100% rename from java/ql/src/Telemetry/AutomodelFrameworkModeExtractCandidates.ql rename to java/ql/automodel/src/AutomodelFrameworkModeExtractCandidates.ql diff --git a/java/ql/src/Telemetry/AutomodelFrameworkModeExtractNegativeExamples.ql b/java/ql/automodel/src/AutomodelFrameworkModeExtractNegativeExamples.ql similarity index 100% rename from java/ql/src/Telemetry/AutomodelFrameworkModeExtractNegativeExamples.ql rename to java/ql/automodel/src/AutomodelFrameworkModeExtractNegativeExamples.ql diff --git a/java/ql/src/Telemetry/AutomodelFrameworkModeExtractPositiveExamples.ql b/java/ql/automodel/src/AutomodelFrameworkModeExtractPositiveExamples.ql similarity index 100% rename from java/ql/src/Telemetry/AutomodelFrameworkModeExtractPositiveExamples.ql rename to java/ql/automodel/src/AutomodelFrameworkModeExtractPositiveExamples.ql diff --git a/java/ql/src/Telemetry/AutomodelJavaUtil.qll b/java/ql/automodel/src/AutomodelJavaUtil.qll similarity index 100% rename from java/ql/src/Telemetry/AutomodelJavaUtil.qll rename to java/ql/automodel/src/AutomodelJavaUtil.qll diff --git a/java/ql/src/Telemetry/AutomodelSharedCharacteristics.qll b/java/ql/automodel/src/AutomodelSharedCharacteristics.qll similarity index 100% rename from java/ql/src/Telemetry/AutomodelSharedCharacteristics.qll rename to java/ql/automodel/src/AutomodelSharedCharacteristics.qll diff --git a/java/ql/src/Telemetry/AutomodelSharedGetCallable.qll b/java/ql/automodel/src/AutomodelSharedGetCallable.qll similarity index 100% rename from java/ql/src/Telemetry/AutomodelSharedGetCallable.qll rename to java/ql/automodel/src/AutomodelSharedGetCallable.qll diff --git a/java/ql/src/Telemetry/AutomodelSinkModelMrvaQueries.ql b/java/ql/automodel/src/AutomodelSinkModelMrvaQueries.ql similarity index 100% rename from java/ql/src/Telemetry/AutomodelSinkModelMrvaQueries.ql rename to java/ql/automodel/src/AutomodelSinkModelMrvaQueries.ql diff --git a/java/ql/test/query-tests/Telemetry/AutomodelApplicationModeExtraction/AutomodelApplicationModeExtractCandidates.expected b/java/ql/automodel/test/AutomodelApplicationModeExtraction/AutomodelApplicationModeExtractCandidates.expected similarity index 100% rename from java/ql/test/query-tests/Telemetry/AutomodelApplicationModeExtraction/AutomodelApplicationModeExtractCandidates.expected rename to java/ql/automodel/test/AutomodelApplicationModeExtraction/AutomodelApplicationModeExtractCandidates.expected diff --git a/java/ql/test/query-tests/Telemetry/AutomodelApplicationModeExtraction/AutomodelApplicationModeExtractCandidates.qlref b/java/ql/automodel/test/AutomodelApplicationModeExtraction/AutomodelApplicationModeExtractCandidates.qlref similarity index 100% rename from java/ql/test/query-tests/Telemetry/AutomodelApplicationModeExtraction/AutomodelApplicationModeExtractCandidates.qlref rename to java/ql/automodel/test/AutomodelApplicationModeExtraction/AutomodelApplicationModeExtractCandidates.qlref diff --git a/java/ql/test/query-tests/Telemetry/AutomodelApplicationModeExtraction/AutomodelApplicationModeExtractNegativeExamples.expected b/java/ql/automodel/test/AutomodelApplicationModeExtraction/AutomodelApplicationModeExtractNegativeExamples.expected similarity index 100% rename from java/ql/test/query-tests/Telemetry/AutomodelApplicationModeExtraction/AutomodelApplicationModeExtractNegativeExamples.expected rename to java/ql/automodel/test/AutomodelApplicationModeExtraction/AutomodelApplicationModeExtractNegativeExamples.expected diff --git a/java/ql/test/query-tests/Telemetry/AutomodelApplicationModeExtraction/AutomodelApplicationModeExtractNegativeExamples.qlref b/java/ql/automodel/test/AutomodelApplicationModeExtraction/AutomodelApplicationModeExtractNegativeExamples.qlref similarity index 100% rename from java/ql/test/query-tests/Telemetry/AutomodelApplicationModeExtraction/AutomodelApplicationModeExtractNegativeExamples.qlref rename to java/ql/automodel/test/AutomodelApplicationModeExtraction/AutomodelApplicationModeExtractNegativeExamples.qlref diff --git a/java/ql/test/query-tests/Telemetry/AutomodelApplicationModeExtraction/AutomodelApplicationModeExtractPositiveExamples.expected b/java/ql/automodel/test/AutomodelApplicationModeExtraction/AutomodelApplicationModeExtractPositiveExamples.expected similarity index 100% rename from java/ql/test/query-tests/Telemetry/AutomodelApplicationModeExtraction/AutomodelApplicationModeExtractPositiveExamples.expected rename to java/ql/automodel/test/AutomodelApplicationModeExtraction/AutomodelApplicationModeExtractPositiveExamples.expected diff --git a/java/ql/test/query-tests/Telemetry/AutomodelApplicationModeExtraction/AutomodelApplicationModeExtractPositiveExamples.qlref b/java/ql/automodel/test/AutomodelApplicationModeExtraction/AutomodelApplicationModeExtractPositiveExamples.qlref similarity index 100% rename from java/ql/test/query-tests/Telemetry/AutomodelApplicationModeExtraction/AutomodelApplicationModeExtractPositiveExamples.qlref rename to java/ql/automodel/test/AutomodelApplicationModeExtraction/AutomodelApplicationModeExtractPositiveExamples.qlref diff --git a/java/ql/test/query-tests/Telemetry/AutomodelApplicationModeExtraction/Test.java b/java/ql/automodel/test/AutomodelApplicationModeExtraction/Test.java similarity index 100% rename from java/ql/test/query-tests/Telemetry/AutomodelApplicationModeExtraction/Test.java rename to java/ql/automodel/test/AutomodelApplicationModeExtraction/Test.java diff --git a/java/ql/test/query-tests/Telemetry/AutomodelFrameworkModeExtraction/AutomodelFrameworkModeExtractCandidates.expected b/java/ql/automodel/test/AutomodelFrameworkModeExtraction/AutomodelFrameworkModeExtractCandidates.expected similarity index 100% rename from java/ql/test/query-tests/Telemetry/AutomodelFrameworkModeExtraction/AutomodelFrameworkModeExtractCandidates.expected rename to java/ql/automodel/test/AutomodelFrameworkModeExtraction/AutomodelFrameworkModeExtractCandidates.expected diff --git a/java/ql/test/query-tests/Telemetry/AutomodelFrameworkModeExtraction/AutomodelFrameworkModeExtractCandidates.qlref b/java/ql/automodel/test/AutomodelFrameworkModeExtraction/AutomodelFrameworkModeExtractCandidates.qlref similarity index 100% rename from java/ql/test/query-tests/Telemetry/AutomodelFrameworkModeExtraction/AutomodelFrameworkModeExtractCandidates.qlref rename to java/ql/automodel/test/AutomodelFrameworkModeExtraction/AutomodelFrameworkModeExtractCandidates.qlref diff --git a/java/ql/test/query-tests/Telemetry/AutomodelFrameworkModeExtraction/AutomodelFrameworkModeExtractNegativeExamples.expected b/java/ql/automodel/test/AutomodelFrameworkModeExtraction/AutomodelFrameworkModeExtractNegativeExamples.expected similarity index 100% rename from java/ql/test/query-tests/Telemetry/AutomodelFrameworkModeExtraction/AutomodelFrameworkModeExtractNegativeExamples.expected rename to java/ql/automodel/test/AutomodelFrameworkModeExtraction/AutomodelFrameworkModeExtractNegativeExamples.expected diff --git a/java/ql/test/query-tests/Telemetry/AutomodelFrameworkModeExtraction/AutomodelFrameworkModeExtractNegativeExamples.qlref b/java/ql/automodel/test/AutomodelFrameworkModeExtraction/AutomodelFrameworkModeExtractNegativeExamples.qlref similarity index 100% rename from java/ql/test/query-tests/Telemetry/AutomodelFrameworkModeExtraction/AutomodelFrameworkModeExtractNegativeExamples.qlref rename to java/ql/automodel/test/AutomodelFrameworkModeExtraction/AutomodelFrameworkModeExtractNegativeExamples.qlref diff --git a/java/ql/test/query-tests/Telemetry/AutomodelFrameworkModeExtraction/AutomodelFrameworkModeExtractPositiveExamples.expected b/java/ql/automodel/test/AutomodelFrameworkModeExtraction/AutomodelFrameworkModeExtractPositiveExamples.expected similarity index 100% rename from java/ql/test/query-tests/Telemetry/AutomodelFrameworkModeExtraction/AutomodelFrameworkModeExtractPositiveExamples.expected rename to java/ql/automodel/test/AutomodelFrameworkModeExtraction/AutomodelFrameworkModeExtractPositiveExamples.expected diff --git a/java/ql/test/query-tests/Telemetry/AutomodelFrameworkModeExtraction/AutomodelFrameworkModeExtractPositiveExamples.qlref b/java/ql/automodel/test/AutomodelFrameworkModeExtraction/AutomodelFrameworkModeExtractPositiveExamples.qlref similarity index 100% rename from java/ql/test/query-tests/Telemetry/AutomodelFrameworkModeExtraction/AutomodelFrameworkModeExtractPositiveExamples.qlref rename to java/ql/automodel/test/AutomodelFrameworkModeExtraction/AutomodelFrameworkModeExtractPositiveExamples.qlref diff --git a/java/ql/test/query-tests/Telemetry/AutomodelFrameworkModeExtraction/com/github/codeql/test/NonPublicClass.java b/java/ql/automodel/test/AutomodelFrameworkModeExtraction/com/github/codeql/test/NonPublicClass.java similarity index 100% rename from java/ql/test/query-tests/Telemetry/AutomodelFrameworkModeExtraction/com/github/codeql/test/NonPublicClass.java rename to java/ql/automodel/test/AutomodelFrameworkModeExtraction/com/github/codeql/test/NonPublicClass.java diff --git a/java/ql/test/query-tests/Telemetry/AutomodelFrameworkModeExtraction/com/github/codeql/test/PublicClass.java b/java/ql/automodel/test/AutomodelFrameworkModeExtraction/com/github/codeql/test/PublicClass.java similarity index 100% rename from java/ql/test/query-tests/Telemetry/AutomodelFrameworkModeExtraction/com/github/codeql/test/PublicClass.java rename to java/ql/automodel/test/AutomodelFrameworkModeExtraction/com/github/codeql/test/PublicClass.java diff --git a/java/ql/test/query-tests/Telemetry/AutomodelFrameworkModeExtraction/com/github/codeql/test/PublicInterface.java b/java/ql/automodel/test/AutomodelFrameworkModeExtraction/com/github/codeql/test/PublicInterface.java similarity index 100% rename from java/ql/test/query-tests/Telemetry/AutomodelFrameworkModeExtraction/com/github/codeql/test/PublicInterface.java rename to java/ql/automodel/test/AutomodelFrameworkModeExtraction/com/github/codeql/test/PublicInterface.java diff --git a/java/ql/test/query-tests/Telemetry/AutomodelFrameworkModeExtraction/java/io/File.java b/java/ql/automodel/test/AutomodelFrameworkModeExtraction/java/io/File.java similarity index 100% rename from java/ql/test/query-tests/Telemetry/AutomodelFrameworkModeExtraction/java/io/File.java rename to java/ql/automodel/test/AutomodelFrameworkModeExtraction/java/io/File.java diff --git a/java/ql/test/query-tests/Telemetry/AutomodelFrameworkModeExtraction/java/nio/file/Files.java b/java/ql/automodel/test/AutomodelFrameworkModeExtraction/java/nio/file/Files.java similarity index 100% rename from java/ql/test/query-tests/Telemetry/AutomodelFrameworkModeExtraction/java/nio/file/Files.java rename to java/ql/automodel/test/AutomodelFrameworkModeExtraction/java/nio/file/Files.java From e7ca2330cbfdc454437ce5ee15ed52d495694c9d Mon Sep 17 00:00:00 2001 From: Anders Starcke Henriksen Date: Thu, 3 Aug 2023 15:55:42 +0200 Subject: [PATCH 002/122] Update workspace. --- codeql-workspace.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/codeql-workspace.yml b/codeql-workspace.yml index c2258bd1363..042c5787843 100644 --- a/codeql-workspace.yml +++ b/codeql-workspace.yml @@ -4,6 +4,7 @@ provide: - "*/ql/test/qlpack.yml" - "*/ql/examples/qlpack.yml" - "*/ql/consistency-queries/qlpack.yml" + - "*/ql/automodel/qlpack.yml" - "shared/*/qlpack.yml" - "cpp/ql/test/query-tests/Security/CWE/CWE-190/semmle/tainted/qlpack.yml" - "go/ql/config/legacy-support/qlpack.yml" From 3ef82c10919d0b51253acdfdf17f6d680ad94710 Mon Sep 17 00:00:00 2001 From: Anders Starcke Henriksen Date: Fri, 4 Aug 2023 10:22:17 +0200 Subject: [PATCH 003/122] Address comments. --- codeql-workspace.yml | 3 ++- java/ql/automodel/publish.sh | 28 ++++++++++++++++++++++++++ java/ql/automodel/{ => src}/qlpack.yml | 7 ++++--- java/ql/automodel/test/qlpack.yml | 12 +++++++++++ 4 files changed, 46 insertions(+), 4 deletions(-) create mode 100755 java/ql/automodel/publish.sh rename java/ql/automodel/{ => src}/qlpack.yml (51%) create mode 100644 java/ql/automodel/test/qlpack.yml diff --git a/codeql-workspace.yml b/codeql-workspace.yml index 042c5787843..2d86498cbea 100644 --- a/codeql-workspace.yml +++ b/codeql-workspace.yml @@ -4,7 +4,8 @@ provide: - "*/ql/test/qlpack.yml" - "*/ql/examples/qlpack.yml" - "*/ql/consistency-queries/qlpack.yml" - - "*/ql/automodel/qlpack.yml" + - "*/ql/automodel/src/qlpack.yml" + - "*/ql/automodel/test/qlpack.yml" - "shared/*/qlpack.yml" - "cpp/ql/test/query-tests/Security/CWE/CWE-190/semmle/tainted/qlpack.yml" - "go/ql/config/legacy-support/qlpack.yml" diff --git a/java/ql/automodel/publish.sh b/java/ql/automodel/publish.sh new file mode 100755 index 00000000000..04b0d3f6ac1 --- /dev/null +++ b/java/ql/automodel/publish.sh @@ -0,0 +1,28 @@ +#!/bin/sh +set -e + +AUTOMODEL_ROOT="$(dirname $0)" +WORKSPACE_ROOT="$AUTOMODEL_ROOT/../../../.." +GRPS="automodel,-test" + +if [ -z "$CODEQL_DIST" ]; then + echo "CODEQL_DIST not set" + exit -1 +fi + +cd "$AUTOMODEL_ROOT" +echo Testing automodel queries +"${CODEQL_DIST}/codeql" test run test + +cd "$WORKSPACE_ROOT" + +echo Preparing release +"${CODEQL_DIST}/codeql" pack release --groups $GRPS + +echo Publishing automodel +"${CODEQL_DIST}/codeql" pack publish --groups $GRPS + +echo Bumping versions +"${CODEQL_DIST}/codeql" pack post-release --groups $GRPS + +echo Automodel packs successfully published. Please commit and push the version changes. diff --git a/java/ql/automodel/qlpack.yml b/java/ql/automodel/src/qlpack.yml similarity index 51% rename from java/ql/automodel/qlpack.yml rename to java/ql/automodel/src/qlpack.yml index 12d756bc67d..9a6f54562ef 100644 --- a/java/ql/automodel/qlpack.yml +++ b/java/ql/automodel/src/qlpack.yml @@ -1,7 +1,8 @@ name: codeql/java-automodel-queries -version: 0.0.1 +version: 0.0.1-dev +groups: + - java + - automodel dependencies: codeql/java-all: ${workspace} -extractor: java -tests: test warnOnImplicitThis: true \ No newline at end of file diff --git a/java/ql/automodel/test/qlpack.yml b/java/ql/automodel/test/qlpack.yml new file mode 100644 index 00000000000..11f5ec8c192 --- /dev/null +++ b/java/ql/automodel/test/qlpack.yml @@ -0,0 +1,12 @@ +name: codeql/java-automodel-tests +version: 0.0.1-dev +groups: + - java + - automodel + - test +dependencies: + codeql/java-all: ${workspace} + codeql/java-automodel-queries: ${workspace} +extractor: java +tests: . +warnOnImplicitThis: true \ No newline at end of file From 0d78eeb871626346b00e78b06d79979412e1906f Mon Sep 17 00:00:00 2001 From: Anders Starcke Henriksen Date: Mon, 7 Aug 2023 10:47:59 +0200 Subject: [PATCH 004/122] Address comments. --- csharp/ql/campaigns/Solorigate/publish.sh | 2 +- java/ql/automodel/publish.sh | 2 +- .../AutomodelApplicationModeExtractCandidates.qlref | 2 +- .../AutomodelApplicationModeExtractNegativeExamples.qlref | 2 +- .../AutomodelApplicationModeExtractPositiveExamples.qlref | 2 +- .../AutomodelFrameworkModeExtractCandidates.qlref | 2 +- .../AutomodelFrameworkModeExtractNegativeExamples.qlref | 2 +- .../AutomodelFrameworkModeExtractPositiveExamples.qlref | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/csharp/ql/campaigns/Solorigate/publish.sh b/csharp/ql/campaigns/Solorigate/publish.sh index 2d1dcfa3bfb..0c57fd6c488 100755 --- a/csharp/ql/campaigns/Solorigate/publish.sh +++ b/csharp/ql/campaigns/Solorigate/publish.sh @@ -1,7 +1,7 @@ #!/bin/sh set -e -SOLORIGATE_ROOT="$(dirname $0)" +SOLORIGATE_ROOT="$(readlink -f "$(dirname $0)")" WORKSPACE_ROOT="$SOLORIGATE_ROOT/../../../.." GRPS="solorigate,-test" diff --git a/java/ql/automodel/publish.sh b/java/ql/automodel/publish.sh index 04b0d3f6ac1..202cdd690a7 100755 --- a/java/ql/automodel/publish.sh +++ b/java/ql/automodel/publish.sh @@ -1,7 +1,7 @@ #!/bin/sh set -e -AUTOMODEL_ROOT="$(dirname $0)" +AUTOMODEL_ROOT="$(readlink -f "$(dirname $0)")" WORKSPACE_ROOT="$AUTOMODEL_ROOT/../../../.." GRPS="automodel,-test" diff --git a/java/ql/automodel/test/AutomodelApplicationModeExtraction/AutomodelApplicationModeExtractCandidates.qlref b/java/ql/automodel/test/AutomodelApplicationModeExtraction/AutomodelApplicationModeExtractCandidates.qlref index b97c87be55f..5ea712b0239 100644 --- a/java/ql/automodel/test/AutomodelApplicationModeExtraction/AutomodelApplicationModeExtractCandidates.qlref +++ b/java/ql/automodel/test/AutomodelApplicationModeExtraction/AutomodelApplicationModeExtractCandidates.qlref @@ -1 +1 @@ -Telemetry/AutomodelApplicationModeExtractCandidates.ql +AutomodelApplicationModeExtractCandidates.ql diff --git a/java/ql/automodel/test/AutomodelApplicationModeExtraction/AutomodelApplicationModeExtractNegativeExamples.qlref b/java/ql/automodel/test/AutomodelApplicationModeExtraction/AutomodelApplicationModeExtractNegativeExamples.qlref index 43610d3cc1a..ccd3827571e 100644 --- a/java/ql/automodel/test/AutomodelApplicationModeExtraction/AutomodelApplicationModeExtractNegativeExamples.qlref +++ b/java/ql/automodel/test/AutomodelApplicationModeExtraction/AutomodelApplicationModeExtractNegativeExamples.qlref @@ -1 +1 @@ -Telemetry/AutomodelApplicationModeExtractNegativeExamples.ql +AutomodelApplicationModeExtractNegativeExamples.ql diff --git a/java/ql/automodel/test/AutomodelApplicationModeExtraction/AutomodelApplicationModeExtractPositiveExamples.qlref b/java/ql/automodel/test/AutomodelApplicationModeExtraction/AutomodelApplicationModeExtractPositiveExamples.qlref index 585a78f94f9..5cf6baa1dba 100644 --- a/java/ql/automodel/test/AutomodelApplicationModeExtraction/AutomodelApplicationModeExtractPositiveExamples.qlref +++ b/java/ql/automodel/test/AutomodelApplicationModeExtraction/AutomodelApplicationModeExtractPositiveExamples.qlref @@ -1 +1 @@ -Telemetry/AutomodelApplicationModeExtractPositiveExamples.ql +AutomodelApplicationModeExtractPositiveExamples.ql diff --git a/java/ql/automodel/test/AutomodelFrameworkModeExtraction/AutomodelFrameworkModeExtractCandidates.qlref b/java/ql/automodel/test/AutomodelFrameworkModeExtraction/AutomodelFrameworkModeExtractCandidates.qlref index e68551eb3ec..e9b513032aa 100644 --- a/java/ql/automodel/test/AutomodelFrameworkModeExtraction/AutomodelFrameworkModeExtractCandidates.qlref +++ b/java/ql/automodel/test/AutomodelFrameworkModeExtraction/AutomodelFrameworkModeExtractCandidates.qlref @@ -1 +1 @@ -Telemetry/AutomodelFrameworkModeExtractCandidates.ql +AutomodelFrameworkModeExtractCandidates.ql diff --git a/java/ql/automodel/test/AutomodelFrameworkModeExtraction/AutomodelFrameworkModeExtractNegativeExamples.qlref b/java/ql/automodel/test/AutomodelFrameworkModeExtraction/AutomodelFrameworkModeExtractNegativeExamples.qlref index d58a0997fdc..69498caf899 100644 --- a/java/ql/automodel/test/AutomodelFrameworkModeExtraction/AutomodelFrameworkModeExtractNegativeExamples.qlref +++ b/java/ql/automodel/test/AutomodelFrameworkModeExtraction/AutomodelFrameworkModeExtractNegativeExamples.qlref @@ -1 +1 @@ -Telemetry/AutomodelFrameworkModeExtractNegativeExamples.ql +AutomodelFrameworkModeExtractNegativeExamples.ql diff --git a/java/ql/automodel/test/AutomodelFrameworkModeExtraction/AutomodelFrameworkModeExtractPositiveExamples.qlref b/java/ql/automodel/test/AutomodelFrameworkModeExtraction/AutomodelFrameworkModeExtractPositiveExamples.qlref index ae52dc66496..fbda3f3dd37 100644 --- a/java/ql/automodel/test/AutomodelFrameworkModeExtraction/AutomodelFrameworkModeExtractPositiveExamples.qlref +++ b/java/ql/automodel/test/AutomodelFrameworkModeExtraction/AutomodelFrameworkModeExtractPositiveExamples.qlref @@ -1 +1 @@ -Telemetry/AutomodelFrameworkModeExtractPositiveExamples.ql +AutomodelFrameworkModeExtractPositiveExamples.ql From 3918e57ffe226106021853c3f9573ad9142fa41a Mon Sep 17 00:00:00 2001 From: Anders Starcke Henriksen Date: Tue, 8 Aug 2023 15:10:12 +0200 Subject: [PATCH 005/122] Take filter pack into account. --- .../Telemetry => automodel/src}/AutomodelCandidateFilter.yml | 0 java/ql/automodel/src/qlpack.yml | 2 ++ java/ql/src/qlpack.yml | 1 - 3 files changed, 2 insertions(+), 1 deletion(-) rename java/ql/{src/Telemetry => automodel/src}/AutomodelCandidateFilter.yml (100%) diff --git a/java/ql/src/Telemetry/AutomodelCandidateFilter.yml b/java/ql/automodel/src/AutomodelCandidateFilter.yml similarity index 100% rename from java/ql/src/Telemetry/AutomodelCandidateFilter.yml rename to java/ql/automodel/src/AutomodelCandidateFilter.yml diff --git a/java/ql/automodel/src/qlpack.yml b/java/ql/automodel/src/qlpack.yml index 9a6f54562ef..1ae0225cb2a 100644 --- a/java/ql/automodel/src/qlpack.yml +++ b/java/ql/automodel/src/qlpack.yml @@ -5,4 +5,6 @@ groups: - automodel dependencies: codeql/java-all: ${workspace} +dataExtensions: + - AutomodelCandidateFilter.yml warnOnImplicitThis: true \ No newline at end of file diff --git a/java/ql/src/qlpack.yml b/java/ql/src/qlpack.yml index 0e7c892f469..76f4fcc7797 100644 --- a/java/ql/src/qlpack.yml +++ b/java/ql/src/qlpack.yml @@ -12,5 +12,4 @@ dependencies: codeql/util: ${workspace} dataExtensions: - Telemetry/ExtractorInformation.yml - - Telemetry/AutomodelCandidateFilter.yml warnOnImplicitThis: true From 885e25ff2d856c035c48e73878ce3a8d1e695902 Mon Sep 17 00:00:00 2001 From: Rasmus Lerchedahl Petersen Date: Wed, 9 Aug 2023 14:25:33 +0200 Subject: [PATCH 006/122] Python: use file-name-convention of `*.model.yml` --- python/ql/lib/qlpack.yml | 2 +- .../frameworks/data/internal/{model.yml => empty.model.yml} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename python/ql/lib/semmle/python/frameworks/data/internal/{model.yml => empty.model.yml} (100%) diff --git a/python/ql/lib/qlpack.yml b/python/ql/lib/qlpack.yml index 8d8f85f53ac..3d00eb3e4e6 100644 --- a/python/ql/lib/qlpack.yml +++ b/python/ql/lib/qlpack.yml @@ -13,5 +13,5 @@ dependencies: codeql/util: ${workspace} codeql/yaml: ${workspace} dataExtensions: - - semmle/python/frameworks/**/model.yml + - semmle/python/frameworks/**/*.model.yml warnOnImplicitThis: true diff --git a/python/ql/lib/semmle/python/frameworks/data/internal/model.yml b/python/ql/lib/semmle/python/frameworks/data/internal/empty.model.yml similarity index 100% rename from python/ql/lib/semmle/python/frameworks/data/internal/model.yml rename to python/ql/lib/semmle/python/frameworks/data/internal/empty.model.yml From 168a1e01a40ed78134336b0cb3654294064c940d Mon Sep 17 00:00:00 2001 From: Rasmus Lerchedahl Petersen Date: Wed, 9 Aug 2023 21:21:21 +0200 Subject: [PATCH 007/122] Python: move test to data extensions For this test, we can simply use the convention, that a file called `[ql-file-stem].ext.yml` will be used as data extensions exactly for the test represented by `ql-file`. --- .../frameworks/data/test.ext.yml | 76 ++++++++++++++ .../library-tests/frameworks/data/test.ql | 98 ------------------- 2 files changed, 76 insertions(+), 98 deletions(-) create mode 100644 python/ql/test/library-tests/frameworks/data/test.ext.yml diff --git a/python/ql/test/library-tests/frameworks/data/test.ext.yml b/python/ql/test/library-tests/frameworks/data/test.ext.yml new file mode 100644 index 00000000000..f3606ebd49f --- /dev/null +++ b/python/ql/test/library-tests/frameworks/data/test.ext.yml @@ -0,0 +1,76 @@ +extensions: + # Contribute empty data sets to avoid errors about an undefined extensionals + - addsTo: + pack: codeql/python-all + extensible: sourceModel + data: + - ["testlib", "Member[getSource].ReturnValue", "test-source"] + - ["testlib.Alias", "", "test-source"] + # testing parameter syntax + - ["testlib", "Member[Callbacks].Member[first].Argument[0].Parameter[0]", "test-source"] + - ["testlib", "Member[Callbacks].Member[param1to3].Argument[0].Parameter[1..3]", "test-source"] + - ["testlib", "Member[Callbacks].Member[nonFirst].Argument[0].Parameter[1..]", "test-source"] + # Common tokens. + - ["testlib", "Member[CommonTokens].Member[makePromise].ReturnValue.Awaited", "test-source"] + - ["testlib", "Member[CommonTokens].Member[Class].Instance", "test-source"] + - ["testlib", "Member[CommonTokens].Member[Super].Subclass.Instance", "test-source"] + # method + - ["testlib", "Member[CommonTokens].Member[Class].Instance.Method[foo]", "test-source"] + # testing non-positional arguments + - ["testlib", "Member[ArgPos].Member[MyClass].Subclass.Member[foo].Parameter[self]", "test-source"] + - ["testlib", "Member[ArgPos].Member[MyClass].Subclass.Member[foo].Parameter[named:]", "test-source"] + - ["testlib", "Member[ArgPos].Member[MyClass].Subclass.Member[secondAndAfter].Parameter[1..]", "test-source"] + - ["testlib", "Member[ArgPos].Member[MyClass].Subclass.Member[otherSelfTest].Parameter[0]", "test-source"] + - ["testlib", "Member[ArgPos].Member[MyClass].Subclass.Member[anyParam].Parameter[any]", "test-source"] + - ["testlib", "Member[ArgPos].Member[MyClass].Subclass.Member[anyNamed].Parameter[any-named]", "test-source"] + + - addsTo: + pack: codeql/python-all + extensible: sinkModel + data: + - ["testlib", "Member[mySink].Argument[0,sinkName:]", "test-sink"] + # testing argument syntax + - ["testlib", "Member[Args].Member[arg0].Argument[0]", "test-sink"] + - ["testlib", "Member[Args].Member[arg1to3].Argument[1..3]", "test-sink"] + - ["testlib", "Member[Args].Member[lastarg].Argument[N-1]", "test-sink"] + - ["testlib", "Member[Args].Member[nonFist].Argument[1..]", "test-sink"] + # callsite filter. + - ["testlib", "Member[CallFilter].Member[arityOne].WithArity[1].Argument[any]", "test-sink"] + - ["testlib", "Member[CallFilter].Member[twoOrMore].WithArity[2..].Argument[0..]", "test-sink"] + # testing non-positional arguments + - ["testlib", "Member[ArgPos].Instance.Member[self_thing].Argument[self]", "test-sink"] + # any argument + - ["testlib", "Member[ArgPos].Member[anyParam].Argument[any]", "test-sink"] + - ["testlib", "Member[ArgPos].Member[anyNamed].Argument[any-named]", "test-sink"] + # testing package syntax + - ["foo1.bar", "Member[baz1].Argument[any]", "test-sink"] + - ["foo2", "Member[bar].Member[baz2].Argument[any]", "test-sink"] + # testing fuzzy + - ["testlib", "Fuzzy.Member[fuzzyCall].Argument[0]", "test-sink"] + # testing syntax errors + - ["testlib", "Member[foo],Member[bar]", "test-sink"] + - ["testlib", "Member[foo] Member[bar]", "test-sink"] + - ["testlib", "Member[foo]. Member[bar]", "test-sink"] + - ["testlib", "Member[foo], Member[bar]", "test-sink"] + - ["testlib", "Member[foo]..Member[bar]", "test-sink"] + - ["testlib", "Member[foo] .Member[bar]", "test-sink"] + - ["testlib", "Member[foo]Member[bar]", "test-sink"] + - ["testlib", "Member[foo", "test-sink"] + - ["testlib", "Member[foo]]", "test-sink"] + - ["testlib", "Member[foo]].Member[bar]", "test-sink"] + + - addsTo: + pack: codeql/python-all + extensible: summaryModel + data: + - ["testlib", "Member[Steps].Member[preserveTaint].Call", "Argument[0]", "ReturnValue", "taint"] + - ["testlib", "Member[Steps].Member[taintIntoCallback]", "Argument[0]", "Argument[1..2].Parameter[0]", "taint"] + - ["testlib", "Member[Steps].Member[preserveArgZeroAndTwo]", "Argument[0,2]", "ReturnValue", "taint"] + - ["testlib", "Member[Steps].Member[preserveAllButFirstArgument].Call", "Argument[1..]", "ReturnValue", "taint"] + + - addsTo: + pack: codeql/python-all + extensible: typeModel + data: + - ["testlib.Alias", "testlib", "Member[alias].ReturnValue"] + - ["testlib.Alias", "testlib.Alias", "Member[chain].ReturnValue"] diff --git a/python/ql/test/library-tests/frameworks/data/test.ql b/python/ql/test/library-tests/frameworks/data/test.ql index d9f270e6caf..6dbda311fa7 100644 --- a/python/ql/test/library-tests/frameworks/data/test.ql +++ b/python/ql/test/library-tests/frameworks/data/test.ql @@ -5,86 +5,6 @@ import semmle.python.dataflow.new.TaintTracking import semmle.python.dataflow.new.DataFlow private import semmle.python.ApiGraphs -class Steps extends ModelInput::SummaryModelCsv { - override predicate row(string row) { - // type;path;input;output;kind - row = - [ - "testlib;Member[Steps].Member[preserveTaint].Call;Argument[0];ReturnValue;taint", - "testlib;Member[Steps].Member[taintIntoCallback];Argument[0];Argument[1..2].Parameter[0];taint", - "testlib;Member[Steps].Member[preserveArgZeroAndTwo];Argument[0,2];ReturnValue;taint", - "testlib;Member[Steps].Member[preserveAllButFirstArgument].Call;Argument[1..];ReturnValue;taint", - ] - } -} - -class Types extends ModelInput::TypeModelCsv { - override predicate row(string row) { - // type1;type2;path - row = - [ - "testlib.Alias;testlib;Member[alias].ReturnValue", - "testlib.Alias;testlib.Alias;Member[chain].ReturnValue", - ] - } -} - -class Sinks extends ModelInput::SinkModelCsv { - override predicate row(string row) { - // type;path;kind - row = - [ - "testlib;Member[mySink].Argument[0,sinkName:];test-sink", - // testing argument syntax - "testlib;Member[Args].Member[arg0].Argument[0];test-sink", // - "testlib;Member[Args].Member[arg1to3].Argument[1..3];test-sink", // - "testlib;Member[Args].Member[lastarg].Argument[N-1];test-sink", // - "testlib;Member[Args].Member[nonFist].Argument[1..];test-sink", // - // callsite filter. - "testlib;Member[CallFilter].Member[arityOne].WithArity[1].Argument[any];test-sink", // - "testlib;Member[CallFilter].Member[twoOrMore].WithArity[2..].Argument[0..];test-sink", // - // testing non-positional arguments - "testlib;Member[ArgPos].Instance.Member[self_thing].Argument[self];test-sink", // - // any argument - "testlib;Member[ArgPos].Member[anyParam].Argument[any];test-sink", // - "testlib;Member[ArgPos].Member[anyNamed].Argument[any-named];test-sink", // - // testing package syntax - "foo1.bar;Member[baz1].Argument[any];test-sink", // - "foo2;Member[bar].Member[baz2].Argument[any];test-sink", // - // testing fuzzy - "testlib;Fuzzy.Member[fuzzyCall].Argument[0];test-sink", // - ] - } -} - -class Sources extends ModelInput::SourceModelCsv { - // type;path;kind - override predicate row(string row) { - row = - [ - "testlib;Member[getSource].ReturnValue;test-source", // - "testlib.Alias;;test-source", - // testing parameter syntax - "testlib;Member[Callbacks].Member[first].Argument[0].Parameter[0];test-source", // - "testlib;Member[Callbacks].Member[param1to3].Argument[0].Parameter[1..3];test-source", // - "testlib;Member[Callbacks].Member[nonFirst].Argument[0].Parameter[1..];test-source", // - // Common tokens. - "testlib;Member[CommonTokens].Member[makePromise].ReturnValue.Awaited;test-source", // - "testlib;Member[CommonTokens].Member[Class].Instance;test-source", // - "testlib;Member[CommonTokens].Member[Super].Subclass.Instance;test-source", // - // method - "testlib;Member[CommonTokens].Member[Class].Instance.Method[foo];test-source", // - // testing non-positional arguments - "testlib;Member[ArgPos].Member[MyClass].Subclass.Member[foo].Parameter[self];test-source", // - "testlib;Member[ArgPos].Member[MyClass].Subclass.Member[foo].Parameter[named:];test-source", // - "testlib;Member[ArgPos].Member[MyClass].Subclass.Member[secondAndAfter].Parameter[1..];test-source", // - "testlib;Member[ArgPos].Member[MyClass].Subclass.Member[otherSelfTest].Parameter[0];test-source", // - "testlib;Member[ArgPos].Member[MyClass].Subclass.Member[anyParam].Parameter[any];test-source", // - "testlib;Member[ArgPos].Member[MyClass].Subclass.Member[anyNamed].Parameter[any-named];test-source", // - ] - } -} - class BasicTaintTracking extends TaintTracking::Configuration { BasicTaintTracking() { this = "BasicTaintTracking" } @@ -109,24 +29,6 @@ query predicate isSource(DataFlow::Node node, string kind) { node = ModelOutput::getASourceNode(kind).asSource() } -class SyntaxErrorTest extends ModelInput::SinkModelCsv { - override predicate row(string row) { - row = - [ - "testlib;Member[foo],Member[bar];test-sink", // - "testlib;Member[foo] Member[bar];test-sink", // - "testlib;Member[foo]. Member[bar];test-sink", // - "testlib;Member[foo], Member[bar];test-sink", // - "testlib;Member[foo]..Member[bar];test-sink", // - "testlib;Member[foo] .Member[bar];test-sink", // - "testlib;Member[foo]Member[bar];test-sink", // - "testlib;Member[foo;test-sink", // - "testlib;Member[foo]];test-sink", // - "testlib;Member[foo]].Member[bar];test-sink", // - ] - } -} - query predicate syntaxErrors(AccessPathSyntax::AccessPath path) { path.hasSyntaxError() } query predicate warning = ModelOutput::getAWarning/0; From dbc60140e034b672d21d64ea072fe020ba668b5c Mon Sep 17 00:00:00 2001 From: Rasmus Lerchedahl Petersen Date: Wed, 9 Aug 2023 20:46:31 +0200 Subject: [PATCH 008/122] Python: move tests to data extensions For these tests, we cannot use the same mechanism, as we want the data extensions to be available for both tests. Instead, we create a ql-pack for the test directory and point to the data entensions from there. This makes the extensions available for all tests in the directory. --- codeql-workspace.yml | 1 + .../dataflow/model-summaries/AllTests.ext.yml | 18 +++++++++++++ .../model-summaries/InlineTaintTest.ql | 1 - .../model-summaries/NormalDataflowTest.ql | 1 - .../model-summaries/TestSummaries.qll | 25 ------------------- .../dataflow/model-summaries/qlpack.yml | 11 ++++++++ 6 files changed, 30 insertions(+), 27 deletions(-) create mode 100644 python/ql/test/experimental/dataflow/model-summaries/AllTests.ext.yml delete mode 100644 python/ql/test/experimental/dataflow/model-summaries/TestSummaries.qll create mode 100644 python/ql/test/experimental/dataflow/model-summaries/qlpack.yml diff --git a/codeql-workspace.yml b/codeql-workspace.yml index c2258bd1363..bff93d79bdb 100644 --- a/codeql-workspace.yml +++ b/codeql-workspace.yml @@ -27,6 +27,7 @@ provide: - "swift/extractor-pack/codeql-extractor.yml" - "swift/integration-tests/qlpack.yml" - "ql/extractor-pack/codeql-extractor.yml" + - "python/ql/test/experimental/dataflow/model-summaries/qlpack.yml" versionPolicies: default: diff --git a/python/ql/test/experimental/dataflow/model-summaries/AllTests.ext.yml b/python/ql/test/experimental/dataflow/model-summaries/AllTests.ext.yml new file mode 100644 index 00000000000..3e14c56d735 --- /dev/null +++ b/python/ql/test/experimental/dataflow/model-summaries/AllTests.ext.yml @@ -0,0 +1,18 @@ +extensions: + - addsTo: + pack: codeql/python-all + extensible: summaryModel + data: + - ["foo", "Member[MS_identity]", "Argument[0]", "ReturnValue", "value"] + - ["foo", "Member[MS_apply_lambda]", "Argument[1]", "Argument[0].Parameter[0]", "value"] + - ["foo", "Member[MS_apply_lambda]", "Argument[0].ReturnValue", "ReturnValue", "value"] + - ["foo", "Member[MS_reversed]", "Argument[0].ListElement", "ReturnValue.ListElement", "value"] + - ["foo", "Member[MS_reversed]", "Argument[0]", "ReturnValue", "taint"] + - ["foo", "Member[MS_list_map]", "Argument[1].ListElement", "Argument[0].Parameter[0]", "value"] + - ["foo", "Member[MS_list_map]", "Argument[0].ReturnValue", "ReturnValue.ListElement", "value"] + - ["foo", "Member[MS_list_map]", "Argument[1]", "ReturnValue", "taint"] + - ["foo", "Member[MS_append_to_list]", "Argument[0].ListElement", "ReturnValue.ListElement", "value"] + - ["foo", "Member[MS_append_to_list]", "Argument[1]", "ReturnValue.ListElement", "value"] + - ["foo", "Member[MS_append_to_list]", "Argument[0]", "ReturnValue", "taint"] + - ["foo", "Member[MS_append_to_list]", "Argument[1]", "ReturnValue", "taint"] + - ["json", "Member[MS_loads]", "Argument[0]", "ReturnValue", "taint"] diff --git a/python/ql/test/experimental/dataflow/model-summaries/InlineTaintTest.ql b/python/ql/test/experimental/dataflow/model-summaries/InlineTaintTest.ql index 551266d7455..dd01b5d3e34 100644 --- a/python/ql/test/experimental/dataflow/model-summaries/InlineTaintTest.ql +++ b/python/ql/test/experimental/dataflow/model-summaries/InlineTaintTest.ql @@ -1,4 +1,3 @@ import python -private import TestSummaries import experimental.meta.InlineTaintTest import MakeInlineTaintTest diff --git a/python/ql/test/experimental/dataflow/model-summaries/NormalDataflowTest.ql b/python/ql/test/experimental/dataflow/model-summaries/NormalDataflowTest.ql index 3e311335e14..3ee344d0b87 100644 --- a/python/ql/test/experimental/dataflow/model-summaries/NormalDataflowTest.ql +++ b/python/ql/test/experimental/dataflow/model-summaries/NormalDataflowTest.ql @@ -1,3 +1,2 @@ import python -private import TestSummaries import experimental.dataflow.TestUtil.NormalDataflowTest diff --git a/python/ql/test/experimental/dataflow/model-summaries/TestSummaries.qll b/python/ql/test/experimental/dataflow/model-summaries/TestSummaries.qll deleted file mode 100644 index 5f1e0a1f90b..00000000000 --- a/python/ql/test/experimental/dataflow/model-summaries/TestSummaries.qll +++ /dev/null @@ -1,25 +0,0 @@ -private import python -private import semmle.python.dataflow.new.FlowSummary -private import semmle.python.frameworks.data.ModelsAsData -private import semmle.python.ApiGraphs - -private class StepsFromModel extends ModelInput::SummaryModelCsv { - override predicate row(string row) { - row = - [ - "foo;Member[MS_identity];Argument[0];ReturnValue;value", - "foo;Member[MS_apply_lambda];Argument[1];Argument[0].Parameter[0];value", - "foo;Member[MS_apply_lambda];Argument[0].ReturnValue;ReturnValue;value", - "foo;Member[MS_reversed];Argument[0].ListElement;ReturnValue.ListElement;value", - "foo;Member[MS_reversed];Argument[0];ReturnValue;taint", - "foo;Member[MS_list_map];Argument[1].ListElement;Argument[0].Parameter[0];value", - "foo;Member[MS_list_map];Argument[0].ReturnValue;ReturnValue.ListElement;value", - "foo;Member[MS_list_map];Argument[1];ReturnValue;taint", - "foo;Member[MS_append_to_list];Argument[0].ListElement;ReturnValue.ListElement;value", - "foo;Member[MS_append_to_list];Argument[1];ReturnValue.ListElement;value", - "foo;Member[MS_append_to_list];Argument[0];ReturnValue;taint", - "foo;Member[MS_append_to_list];Argument[1];ReturnValue;taint", - "json;Member[MS_loads];Argument[0];ReturnValue;taint" - ] - } -} diff --git a/python/ql/test/experimental/dataflow/model-summaries/qlpack.yml b/python/ql/test/experimental/dataflow/model-summaries/qlpack.yml new file mode 100644 index 00000000000..7e1eb0a97cd --- /dev/null +++ b/python/ql/test/experimental/dataflow/model-summaries/qlpack.yml @@ -0,0 +1,11 @@ +name: python-model-summaries-tests +groups: + - python + - test +dependencies: + codeql/python-tests: ${workspace} +extractor: python +tests: . +warnOnImplicitThis: true +dataExtensions: + - AllTests.ext.yml From 94a5aa450ccea0d77c47d8cfdb49d27a897f16a6 Mon Sep 17 00:00:00 2001 From: Geoffrey White <40627776+geoffw0@users.noreply.github.com> Date: Thu, 10 Aug 2023 17:52:01 +0100 Subject: [PATCH 009/122] Swift: Edit the weak sensitive data hashing examples and qhelp to encourage use of HMAC and key derivation algorithms where appropriate. --- .../Security/CWE-328/WeakSensitiveDataHashing.qhelp | 9 +++++++-- .../CWE-328/WeakSensitiveDataHashingBad.swift | 13 +++++++++---- .../CWE-328/WeakSensitiveDataHashingGood.swift | 12 +++++++++--- 3 files changed, 25 insertions(+), 9 deletions(-) diff --git a/swift/ql/src/queries/Security/CWE-328/WeakSensitiveDataHashing.qhelp b/swift/ql/src/queries/Security/CWE-328/WeakSensitiveDataHashing.qhelp index d6806b2ddcc..7e9fe996eeb 100755 --- a/swift/ql/src/queries/Security/CWE-328/WeakSensitiveDataHashing.qhelp +++ b/swift/ql/src/queries/Security/CWE-328/WeakSensitiveDataHashing.qhelp @@ -51,12 +51,17 @@ +

+ Note that special purpose algorithms exist for message authentication (ensuring that a message comes from a particular sender). These algorithms should be used when appropriate, as they address common vulnerabilities of simple hashing schemes in this context. +

+

- The following examples show a function for checking whether the hash - of a certificate matches a known value -- to prevent tampering. + The following examples show a function for fetching data from a + URL along with a hash of the data, perhaps to check the data has + not been tampered with. In the first case the MD5 hashing algorithm is used that is known to be vulnerable to collision attacks.

diff --git a/swift/ql/src/queries/Security/CWE-328/WeakSensitiveDataHashingBad.swift b/swift/ql/src/queries/Security/CWE-328/WeakSensitiveDataHashingBad.swift index a39dd47edce..5153c852342 100755 --- a/swift/ql/src/queries/Security/CWE-328/WeakSensitiveDataHashingBad.swift +++ b/swift/ql/src/queries/Security/CWE-328/WeakSensitiveDataHashingBad.swift @@ -1,5 +1,10 @@ -typealias Hasher = Crypto.Insecure.MD5 +func getContentsAndHash(url: URL) -> (Data, String)? { + guard let data = try? Data(contentsOf: url) else { + return nil + } -func checkCertificate(cert: Array[UInt8], hash: Array[UInt8]) -> Bool - return Hasher.hash(data: cert) == hash // BAD -} + let digest = Insecure.MD5.hash(data: data) + let hash = digest.map { String(format: "%02hhx", $0) }.joined() + + return (data, hash) +} \ No newline at end of file diff --git a/swift/ql/src/queries/Security/CWE-328/WeakSensitiveDataHashingGood.swift b/swift/ql/src/queries/Security/CWE-328/WeakSensitiveDataHashingGood.swift index 7345b2ea49c..4b9e0ec6af3 100755 --- a/swift/ql/src/queries/Security/CWE-328/WeakSensitiveDataHashingGood.swift +++ b/swift/ql/src/queries/Security/CWE-328/WeakSensitiveDataHashingGood.swift @@ -1,4 +1,10 @@ -typealias Hasher = Crypto.SHA512 +func getContentsAndHash(url: URL) -> (Data, String)? { + guard let data = try? Data(contentsOf: url) else { + return nil + } -func checkCertificate(cert: Array[UInt8], hash: Array[UInt8]) -> Bool - return Hasher.hash(data: cert) == hash // GOOD + let digest = SHA512.hash(data: data) + let hash = digest.map { String(format: "%02hhx", $0) }.joined() + + return (data, hash) +} \ No newline at end of file From 551b34e3be78ca397b086c8b7c867e32706e0e54 Mon Sep 17 00:00:00 2001 From: Stephan Brandauer Date: Mon, 14 Aug 2023 11:46:40 +0200 Subject: [PATCH 010/122] Java: Automodel application mode: include candidates that are useful for regression testing --- ...AutomodelApplicationModeExtractCandidates.ql | 16 +++++++++++----- ...delApplicationModeExtractPositiveExamples.ql | 2 +- .../AutomodelSharedCharacteristics.qll | 17 ++++++++++------- ...delApplicationModeExtractCandidates.expected | 7 ++++--- .../Test.java | 3 +-- 5 files changed, 27 insertions(+), 18 deletions(-) diff --git a/java/ql/src/Telemetry/AutomodelApplicationModeExtractCandidates.ql b/java/ql/src/Telemetry/AutomodelApplicationModeExtractCandidates.ql index d58af008d87..b2f6c0f9e3f 100644 --- a/java/ql/src/Telemetry/AutomodelApplicationModeExtractCandidates.ql +++ b/java/ql/src/Telemetry/AutomodelApplicationModeExtractCandidates.ql @@ -55,7 +55,7 @@ private Endpoint getSampleForSignature( from Endpoint endpoint, string message, ApplicationModeMetadataExtractor meta, DollarAtString package, DollarAtString type, DollarAtString subtypes, DollarAtString name, DollarAtString signature, - DollarAtString input, DollarAtString isVarargsArray + DollarAtString input, DollarAtString isVarargsArray, DollarAtString alreadyAiModeled where not exists(CharacteristicsImpl::UninterestingToModelCharacteristic u | u.appliesToEndpoint(endpoint) @@ -67,20 +67,25 @@ where // label it as a sink for one of the sink types of query B, for which it's already a known sink. This would result in // overlap between our detected sinks and the pre-existing modeling. We assume that, if a sink has already been // modeled in a MaD model, then it doesn't belong to any additional sink types, and we don't need to reexamine it. - not CharacteristicsImpl::isSink(endpoint, _, _) and + ( + not CharacteristicsImpl::isSink(endpoint, _, _) and alreadyAiModeled = "false" + or + CharacteristicsImpl::isSink(endpoint, _, any(string s | s.matches("%ai-%"))) and + alreadyAiModeled = "true" + ) and meta.hasMetadata(endpoint, package, type, subtypes, name, signature, input, isVarargsArray) and includeAutomodelCandidate(package, type, name, signature) and // The message is the concatenation of all sink types for which this endpoint is known neither to be a sink nor to be // a non-sink, and we surface only endpoints that have at least one such sink type. message = strictconcat(AutomodelEndpointTypes::SinkType sinkType | - not CharacteristicsImpl::isKnownSink(endpoint, sinkType) and + not CharacteristicsImpl::isKnownSink(endpoint, sinkType, _) and CharacteristicsImpl::isSinkCandidate(endpoint, sinkType) | sinkType, ", " ) select endpoint.asNode(), - message + "\nrelated locations: $@." + "\nmetadata: $@, $@, $@, $@, $@, $@, $@.", // + message + "\nrelated locations: $@." + "\nmetadata: $@, $@, $@, $@, $@, $@, $@, $@.", // CharacteristicsImpl::getRelatedLocationOrCandidate(endpoint, CallContext()), "CallContext", // package, "package", // type, "type", // @@ -88,4 +93,5 @@ select endpoint.asNode(), name, "name", // method name signature, "signature", // input, "input", // - isVarargsArray, "isVarargsArray" + isVarargsArray, "isVarargsArray", // + alreadyAiModeled, "alreadyAiModeled" diff --git a/java/ql/src/Telemetry/AutomodelApplicationModeExtractPositiveExamples.ql b/java/ql/src/Telemetry/AutomodelApplicationModeExtractPositiveExamples.ql index dac9bef0728..7341f512702 100644 --- a/java/ql/src/Telemetry/AutomodelApplicationModeExtractPositiveExamples.ql +++ b/java/ql/src/Telemetry/AutomodelApplicationModeExtractPositiveExamples.ql @@ -22,7 +22,7 @@ where not erroneousEndpoints(endpoint, _, _, _, _, false) and meta.hasMetadata(endpoint, package, type, subtypes, name, signature, input, isVarargsArray) and // Extract positive examples of sinks belonging to the existing ATM query configurations. - CharacteristicsImpl::isKnownSink(endpoint, sinkType) and + CharacteristicsImpl::isKnownSink(endpoint, sinkType, _) and exists(CharacteristicsImpl::getRelatedLocationOrCandidate(endpoint, CallContext())) select endpoint.asNode(), sinkType + "\nrelated locations: $@." + "\nmetadata: $@, $@, $@, $@, $@, $@, $@.", // diff --git a/java/ql/src/Telemetry/AutomodelSharedCharacteristics.qll b/java/ql/src/Telemetry/AutomodelSharedCharacteristics.qll index c1a9a14a10a..845f8385e5b 100644 --- a/java/ql/src/Telemetry/AutomodelSharedCharacteristics.qll +++ b/java/ql/src/Telemetry/AutomodelSharedCharacteristics.qll @@ -94,14 +94,15 @@ module SharedCharacteristics { /** * Holds if `sink` is a known sink of type `endpointType`. */ - predicate isKnownSink(Candidate::Endpoint sink, Candidate::EndpointType endpointType) { + predicate isKnownSink( + Candidate::Endpoint sink, Candidate::EndpointType endpointType, + EndpointCharacteristic characteristic + ) { // If the list of characteristics includes positive indicators with maximal confidence for this class, then it's a // known sink for the class. not endpointType instanceof Candidate::NegativeEndpointType and - exists(EndpointCharacteristic characteristic | - characteristic.appliesToEndpoint(sink) and - characteristic.hasImplications(endpointType, true, maximalConfidence()) - ) + characteristic.appliesToEndpoint(sink) and + characteristic.hasImplications(endpointType, true, maximalConfidence()) } /** @@ -275,15 +276,17 @@ module SharedCharacteristics { private class KnownSinkCharacteristic extends SinkCharacteristic { string madKind; Candidate::EndpointType endpointType; + string provenance; KnownSinkCharacteristic() { Candidate::isKnownKind(madKind, endpointType) and // bind "this" to a unique string differing from that of the SinkType classes - this = madKind + "-characteristic" + this = madKind + "_" + provenance + "_characteristic" and + Candidate::isSink(_, madKind, provenance) } override predicate appliesToEndpoint(Candidate::Endpoint e) { - Candidate::isSink(e, madKind, _) + Candidate::isSink(e, madKind, provenance) } override Candidate::EndpointType getSinkType() { result = endpointType } diff --git a/java/ql/test/query-tests/Telemetry/AutomodelApplicationModeExtraction/AutomodelApplicationModeExtractCandidates.expected b/java/ql/test/query-tests/Telemetry/AutomodelApplicationModeExtraction/AutomodelApplicationModeExtractCandidates.expected index 5f946f49d19..7903223bee6 100644 --- a/java/ql/test/query-tests/Telemetry/AutomodelApplicationModeExtraction/AutomodelApplicationModeExtractCandidates.expected +++ b/java/ql/test/query-tests/Telemetry/AutomodelApplicationModeExtraction/AutomodelApplicationModeExtractCandidates.expected @@ -1,3 +1,4 @@ -| Test.java:16:3:16:11 | reference | command-injection, path-injection, request-forgery, sql-injection\nrelated locations: $@.\nmetadata: $@, $@, $@, $@, $@, $@, $@. | Test.java:16:3:16:24 | set(...) | CallContext | file://java.util.concurrent.atomic:1:1:1:1 | java.util.concurrent.atomic | package | file://AtomicReference:1:1:1:1 | AtomicReference | type | file://false:1:1:1:1 | false | subtypes | file://set:1:1:1:1 | set | name | file://(String):1:1:1:1 | (String) | signature | file://Argument[this]:1:1:1:1 | Argument[this] | input | file://false:1:1:1:1 | false | isVarargsArray | -| Test.java:21:3:21:10 | supplier | command-injection, path-injection, request-forgery, sql-injection\nrelated locations: $@.\nmetadata: $@, $@, $@, $@, $@, $@, $@. | Test.java:21:3:21:16 | get(...) | CallContext | file://java.util.function:1:1:1:1 | java.util.function | package | file://Supplier:1:1:1:1 | Supplier | type | file://true:1:1:1:1 | true | subtypes | file://get:1:1:1:1 | get | name | file://():1:1:1:1 | () | signature | file://Argument[this]:1:1:1:1 | Argument[this] | input | file://false:1:1:1:1 | false | isVarargsArray | -| Test.java:53:4:53:4 | o | command-injection, path-injection, request-forgery, sql-injection\nrelated locations: $@.\nmetadata: $@, $@, $@, $@, $@, $@, $@. | Test.java:51:3:56:3 | walk(...) | CallContext | file://java.nio.file:1:1:1:1 | java.nio.file | package | file://Files:1:1:1:1 | Files | type | file://false:1:1:1:1 | false | subtypes | file://walk:1:1:1:1 | walk | name | file://(Path,FileVisitOption[]):1:1:1:1 | (Path,FileVisitOption[]) | signature | file://Argument[1]:1:1:1:1 | Argument[1] | input | file://true:1:1:1:1 | true | isVarargsArray | +| Test.java:16:3:16:11 | reference | command-injection, path-injection, request-forgery, sql-injection\nrelated locations: $@.\nmetadata: $@, $@, $@, $@, $@, $@, $@, $@. | Test.java:16:3:16:24 | set(...) | CallContext | file://java.util.concurrent.atomic:1:1:1:1 | java.util.concurrent.atomic | package | file://AtomicReference:1:1:1:1 | AtomicReference | type | file://false:1:1:1:1 | false | subtypes | file://set:1:1:1:1 | set | name | file://(String):1:1:1:1 | (String) | signature | file://Argument[this]:1:1:1:1 | Argument[this] | input | file://false:1:1:1:1 | false | isVarargsArray | file://false:1:1:1:1 | false | alreadyAiModeled | +| Test.java:21:3:21:10 | supplier | command-injection, path-injection, request-forgery, sql-injection\nrelated locations: $@.\nmetadata: $@, $@, $@, $@, $@, $@, $@, $@. | Test.java:21:3:21:16 | get(...) | CallContext | file://java.util.function:1:1:1:1 | java.util.function | package | file://Supplier:1:1:1:1 | Supplier | type | file://true:1:1:1:1 | true | subtypes | file://get:1:1:1:1 | get | name | file://():1:1:1:1 | () | signature | file://Argument[this]:1:1:1:1 | Argument[this] | input | file://false:1:1:1:1 | false | isVarargsArray | file://false:1:1:1:1 | false | alreadyAiModeled | +| Test.java:34:4:34:11 | openPath | command-injection, request-forgery, sql-injection\nrelated locations: $@.\nmetadata: $@, $@, $@, $@, $@, $@, $@, $@. | Test.java:33:10:35:3 | newInputStream(...) | CallContext | file://java.nio.file:1:1:1:1 | java.nio.file | package | file://Files:1:1:1:1 | Files | type | file://false:1:1:1:1 | false | subtypes | file://newInputStream:1:1:1:1 | newInputStream | name | file://(Path,OpenOption[]):1:1:1:1 | (Path,OpenOption[]) | signature | file://Argument[0]:1:1:1:1 | Argument[0] | input | file://false:1:1:1:1 | false | isVarargsArray | file://true:1:1:1:1 | true | alreadyAiModeled | +| Test.java:53:4:53:4 | o | command-injection, path-injection, request-forgery, sql-injection\nrelated locations: $@.\nmetadata: $@, $@, $@, $@, $@, $@, $@, $@. | Test.java:51:3:56:3 | walk(...) | CallContext | file://java.nio.file:1:1:1:1 | java.nio.file | package | file://Files:1:1:1:1 | Files | type | file://false:1:1:1:1 | false | subtypes | file://walk:1:1:1:1 | walk | name | file://(Path,FileVisitOption[]):1:1:1:1 | (Path,FileVisitOption[]) | signature | file://Argument[1]:1:1:1:1 | Argument[1] | input | file://true:1:1:1:1 | true | isVarargsArray | file://false:1:1:1:1 | false | alreadyAiModeled | diff --git a/java/ql/test/query-tests/Telemetry/AutomodelApplicationModeExtraction/Test.java b/java/ql/test/query-tests/Telemetry/AutomodelApplicationModeExtraction/Test.java index 1b5cea4b907..1a660f7752f 100644 --- a/java/ql/test/query-tests/Telemetry/AutomodelApplicationModeExtraction/Test.java +++ b/java/ql/test/query-tests/Telemetry/AutomodelApplicationModeExtraction/Test.java @@ -31,7 +31,7 @@ class Test { public static InputStream getInputStream(Path openPath) throws Exception { return Files.newInputStream( - openPath // positive example (known sink) + openPath // positive example (known sink), candidate ("only" ai-modeled, and useful as a candidate in regression testing) ); } @@ -56,4 +56,3 @@ class Test { ); } } - From bc55afcee786d5a59153735cb836604c1768c66a Mon Sep 17 00:00:00 2001 From: Stephan Brandauer Date: Mon, 14 Aug 2023 13:17:55 +0200 Subject: [PATCH 011/122] Java: Automodel framework mode: use new interface --- .../ql/src/Telemetry/AutomodelFrameworkModeExtractCandidates.ql | 2 +- .../Telemetry/AutomodelFrameworkModeExtractPositiveExamples.ql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/java/ql/src/Telemetry/AutomodelFrameworkModeExtractCandidates.ql b/java/ql/src/Telemetry/AutomodelFrameworkModeExtractCandidates.ql index 028a27a9bdc..073cae83a1d 100644 --- a/java/ql/src/Telemetry/AutomodelFrameworkModeExtractCandidates.ql +++ b/java/ql/src/Telemetry/AutomodelFrameworkModeExtractCandidates.ql @@ -35,7 +35,7 @@ where // a non-sink, and we surface only endpoints that have at least one such sink type. message = strictconcat(AutomodelEndpointTypes::SinkType sinkType | - not CharacteristicsImpl::isKnownSink(endpoint, sinkType) and + not CharacteristicsImpl::isKnownSink(endpoint, sinkType, _) and CharacteristicsImpl::isSinkCandidate(endpoint, sinkType) | sinkType, ", " diff --git a/java/ql/src/Telemetry/AutomodelFrameworkModeExtractPositiveExamples.ql b/java/ql/src/Telemetry/AutomodelFrameworkModeExtractPositiveExamples.ql index 2547239ee91..3807de45c5a 100644 --- a/java/ql/src/Telemetry/AutomodelFrameworkModeExtractPositiveExamples.ql +++ b/java/ql/src/Telemetry/AutomodelFrameworkModeExtractPositiveExamples.ql @@ -22,7 +22,7 @@ where not erroneousEndpoints(endpoint, _, _, _, _, false) and meta.hasMetadata(endpoint, package, type, subtypes, name, signature, input, parameterName) and // Extract positive examples of sinks belonging to the existing ATM query configurations. - CharacteristicsImpl::isKnownSink(endpoint, sinkType) + CharacteristicsImpl::isKnownSink(endpoint, sinkType, _) select endpoint, sinkType + "\nrelated locations: $@, $@." + "\nmetadata: $@, $@, $@, $@, $@, $@, $@.", // CharacteristicsImpl::getRelatedLocationOrCandidate(endpoint, MethodDoc()), "MethodDoc", // From 808dc3e8d306dcb6441f27bb3c322917d978c8d6 Mon Sep 17 00:00:00 2001 From: Stephan Brandauer Date: Wed, 16 Aug 2023 09:25:03 +0200 Subject: [PATCH 012/122] Java: Automodel framework mode: track exact ai- provenance in alreadyAiModeled meta data property --- .../AutomodelApplicationModeExtractCandidates.ql | 6 +++--- .../AutomodelApplicationModeExtractCandidates.expected | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/java/ql/src/Telemetry/AutomodelApplicationModeExtractCandidates.ql b/java/ql/src/Telemetry/AutomodelApplicationModeExtractCandidates.ql index b2f6c0f9e3f..b46a25b088d 100644 --- a/java/ql/src/Telemetry/AutomodelApplicationModeExtractCandidates.ql +++ b/java/ql/src/Telemetry/AutomodelApplicationModeExtractCandidates.ql @@ -68,10 +68,10 @@ where // overlap between our detected sinks and the pre-existing modeling. We assume that, if a sink has already been // modeled in a MaD model, then it doesn't belong to any additional sink types, and we don't need to reexamine it. ( - not CharacteristicsImpl::isSink(endpoint, _, _) and alreadyAiModeled = "false" + not CharacteristicsImpl::isSink(endpoint, _, _) and alreadyAiModeled = "" or - CharacteristicsImpl::isSink(endpoint, _, any(string s | s.matches("%ai-%"))) and - alreadyAiModeled = "true" + alreadyAiModeled.matches("%ai-%") and + CharacteristicsImpl::isSink(endpoint, _, alreadyAiModeled) ) and meta.hasMetadata(endpoint, package, type, subtypes, name, signature, input, isVarargsArray) and includeAutomodelCandidate(package, type, name, signature) and diff --git a/java/ql/test/query-tests/Telemetry/AutomodelApplicationModeExtraction/AutomodelApplicationModeExtractCandidates.expected b/java/ql/test/query-tests/Telemetry/AutomodelApplicationModeExtraction/AutomodelApplicationModeExtractCandidates.expected index 7903223bee6..77d51830c67 100644 --- a/java/ql/test/query-tests/Telemetry/AutomodelApplicationModeExtraction/AutomodelApplicationModeExtractCandidates.expected +++ b/java/ql/test/query-tests/Telemetry/AutomodelApplicationModeExtraction/AutomodelApplicationModeExtractCandidates.expected @@ -1,4 +1,4 @@ -| Test.java:16:3:16:11 | reference | command-injection, path-injection, request-forgery, sql-injection\nrelated locations: $@.\nmetadata: $@, $@, $@, $@, $@, $@, $@, $@. | Test.java:16:3:16:24 | set(...) | CallContext | file://java.util.concurrent.atomic:1:1:1:1 | java.util.concurrent.atomic | package | file://AtomicReference:1:1:1:1 | AtomicReference | type | file://false:1:1:1:1 | false | subtypes | file://set:1:1:1:1 | set | name | file://(String):1:1:1:1 | (String) | signature | file://Argument[this]:1:1:1:1 | Argument[this] | input | file://false:1:1:1:1 | false | isVarargsArray | file://false:1:1:1:1 | false | alreadyAiModeled | -| Test.java:21:3:21:10 | supplier | command-injection, path-injection, request-forgery, sql-injection\nrelated locations: $@.\nmetadata: $@, $@, $@, $@, $@, $@, $@, $@. | Test.java:21:3:21:16 | get(...) | CallContext | file://java.util.function:1:1:1:1 | java.util.function | package | file://Supplier:1:1:1:1 | Supplier | type | file://true:1:1:1:1 | true | subtypes | file://get:1:1:1:1 | get | name | file://():1:1:1:1 | () | signature | file://Argument[this]:1:1:1:1 | Argument[this] | input | file://false:1:1:1:1 | false | isVarargsArray | file://false:1:1:1:1 | false | alreadyAiModeled | -| Test.java:34:4:34:11 | openPath | command-injection, request-forgery, sql-injection\nrelated locations: $@.\nmetadata: $@, $@, $@, $@, $@, $@, $@, $@. | Test.java:33:10:35:3 | newInputStream(...) | CallContext | file://java.nio.file:1:1:1:1 | java.nio.file | package | file://Files:1:1:1:1 | Files | type | file://false:1:1:1:1 | false | subtypes | file://newInputStream:1:1:1:1 | newInputStream | name | file://(Path,OpenOption[]):1:1:1:1 | (Path,OpenOption[]) | signature | file://Argument[0]:1:1:1:1 | Argument[0] | input | file://false:1:1:1:1 | false | isVarargsArray | file://true:1:1:1:1 | true | alreadyAiModeled | -| Test.java:53:4:53:4 | o | command-injection, path-injection, request-forgery, sql-injection\nrelated locations: $@.\nmetadata: $@, $@, $@, $@, $@, $@, $@, $@. | Test.java:51:3:56:3 | walk(...) | CallContext | file://java.nio.file:1:1:1:1 | java.nio.file | package | file://Files:1:1:1:1 | Files | type | file://false:1:1:1:1 | false | subtypes | file://walk:1:1:1:1 | walk | name | file://(Path,FileVisitOption[]):1:1:1:1 | (Path,FileVisitOption[]) | signature | file://Argument[1]:1:1:1:1 | Argument[1] | input | file://true:1:1:1:1 | true | isVarargsArray | file://false:1:1:1:1 | false | alreadyAiModeled | +| Test.java:16:3:16:11 | reference | command-injection, path-injection, request-forgery, sql-injection\nrelated locations: $@.\nmetadata: $@, $@, $@, $@, $@, $@, $@, $@. | Test.java:16:3:16:24 | set(...) | CallContext | file://java.util.concurrent.atomic:1:1:1:1 | java.util.concurrent.atomic | package | file://AtomicReference:1:1:1:1 | AtomicReference | type | file://false:1:1:1:1 | false | subtypes | file://set:1:1:1:1 | set | name | file://(String):1:1:1:1 | (String) | signature | file://Argument[this]:1:1:1:1 | Argument[this] | input | file://false:1:1:1:1 | false | isVarargsArray | file://:1:1:1:1 | | alreadyAiModeled | +| Test.java:21:3:21:10 | supplier | command-injection, path-injection, request-forgery, sql-injection\nrelated locations: $@.\nmetadata: $@, $@, $@, $@, $@, $@, $@, $@. | Test.java:21:3:21:16 | get(...) | CallContext | file://java.util.function:1:1:1:1 | java.util.function | package | file://Supplier:1:1:1:1 | Supplier | type | file://true:1:1:1:1 | true | subtypes | file://get:1:1:1:1 | get | name | file://():1:1:1:1 | () | signature | file://Argument[this]:1:1:1:1 | Argument[this] | input | file://false:1:1:1:1 | false | isVarargsArray | file://:1:1:1:1 | | alreadyAiModeled | +| Test.java:34:4:34:11 | openPath | command-injection, request-forgery, sql-injection\nrelated locations: $@.\nmetadata: $@, $@, $@, $@, $@, $@, $@, $@. | Test.java:33:10:35:3 | newInputStream(...) | CallContext | file://java.nio.file:1:1:1:1 | java.nio.file | package | file://Files:1:1:1:1 | Files | type | file://false:1:1:1:1 | false | subtypes | file://newInputStream:1:1:1:1 | newInputStream | name | file://(Path,OpenOption[]):1:1:1:1 | (Path,OpenOption[]) | signature | file://Argument[0]:1:1:1:1 | Argument[0] | input | file://false:1:1:1:1 | false | isVarargsArray | file://ai-manual:1:1:1:1 | ai-manual | alreadyAiModeled | +| Test.java:53:4:53:4 | o | command-injection, path-injection, request-forgery, sql-injection\nrelated locations: $@.\nmetadata: $@, $@, $@, $@, $@, $@, $@, $@. | Test.java:51:3:56:3 | walk(...) | CallContext | file://java.nio.file:1:1:1:1 | java.nio.file | package | file://Files:1:1:1:1 | Files | type | file://false:1:1:1:1 | false | subtypes | file://walk:1:1:1:1 | walk | name | file://(Path,FileVisitOption[]):1:1:1:1 | (Path,FileVisitOption[]) | signature | file://Argument[1]:1:1:1:1 | Argument[1] | input | file://true:1:1:1:1 | true | isVarargsArray | file://:1:1:1:1 | | alreadyAiModeled | From 1b31c4dd4c94f809fbeb5db61caa57023340ad1a Mon Sep 17 00:00:00 2001 From: Anders Starcke Henriksen Date: Thu, 17 Aug 2023 11:07:27 +0200 Subject: [PATCH 013/122] Update filter to point to right pack. --- java/ql/automodel/src/AutomodelCandidateFilter.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/ql/automodel/src/AutomodelCandidateFilter.yml b/java/ql/automodel/src/AutomodelCandidateFilter.yml index 52e64d54446..c945ae3206f 100644 --- a/java/ql/automodel/src/AutomodelCandidateFilter.yml +++ b/java/ql/automodel/src/AutomodelCandidateFilter.yml @@ -1,5 +1,5 @@ extensions: - addsTo: - pack: codeql/java-queries + pack: codeql/java-automodel-queries extensible: automodelCandidateFilter data: [] From 91edde72c4855491edc1593ffdd02c675ee0ffb4 Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Thu, 17 Aug 2023 11:51:00 +0200 Subject: [PATCH 014/122] Python: Port `py/template-injection` to new data-flow I kept all the modeling in _one_ file, since that makes it easy to work with such an external contribution... and I would certainly propose this file setup for the future :+1: --- .../CWE-074/TemplateConstructionConcept.qll | 159 ++++++++++++++++++ .../Security/CWE-074/TemplateInjection.ql | 27 +-- .../TemplateInjectionCustomizations.qll | 51 ++++++ .../CWE-074/TemplateInjectionQuery.qll | 18 ++ .../semmle/python/templates/Airspeed.qll | 27 --- .../semmle/python/templates/Bottle.qll | 48 ------ .../semmle/python/templates/Chameleon.qll | 29 ---- .../semmle/python/templates/Cheetah.qll | 39 ----- .../semmle/python/templates/Chevron.qll | 36 ---- .../python/templates/DjangoTemplate.qll | 35 ---- .../semmle/python/templates/FlaskTemplate.qll | 28 --- .../semmle/python/templates/Genshi.qll | 53 ------ .../semmle/python/templates/Jinja.qll | 49 ------ .../semmle/python/templates/Mako.qll | 27 --- .../semmle/python/templates/SSTISink.qll | 7 - .../semmle/python/templates/Ssti.qll | 13 -- .../semmle/python/templates/TRender.qll | 27 --- .../CWE-074/TemplateInjection.expected | 144 +++++++++------- .../query-tests/Security/CWE-074/options | 1 - 19 files changed, 320 insertions(+), 498 deletions(-) create mode 100644 python/ql/src/experimental/Security/CWE-074/TemplateConstructionConcept.qll create mode 100644 python/ql/src/experimental/Security/CWE-074/TemplateInjectionCustomizations.qll create mode 100644 python/ql/src/experimental/Security/CWE-074/TemplateInjectionQuery.qll delete mode 100644 python/ql/src/experimental/semmle/python/templates/Airspeed.qll delete mode 100644 python/ql/src/experimental/semmle/python/templates/Bottle.qll delete mode 100644 python/ql/src/experimental/semmle/python/templates/Chameleon.qll delete mode 100644 python/ql/src/experimental/semmle/python/templates/Cheetah.qll delete mode 100644 python/ql/src/experimental/semmle/python/templates/Chevron.qll delete mode 100644 python/ql/src/experimental/semmle/python/templates/DjangoTemplate.qll delete mode 100644 python/ql/src/experimental/semmle/python/templates/FlaskTemplate.qll delete mode 100644 python/ql/src/experimental/semmle/python/templates/Genshi.qll delete mode 100644 python/ql/src/experimental/semmle/python/templates/Jinja.qll delete mode 100644 python/ql/src/experimental/semmle/python/templates/Mako.qll delete mode 100644 python/ql/src/experimental/semmle/python/templates/SSTISink.qll delete mode 100644 python/ql/src/experimental/semmle/python/templates/Ssti.qll delete mode 100644 python/ql/src/experimental/semmle/python/templates/TRender.qll delete mode 100644 python/ql/test/experimental/query-tests/Security/CWE-074/options diff --git a/python/ql/src/experimental/Security/CWE-074/TemplateConstructionConcept.qll b/python/ql/src/experimental/Security/CWE-074/TemplateConstructionConcept.qll new file mode 100644 index 00000000000..a20babf15eb --- /dev/null +++ b/python/ql/src/experimental/Security/CWE-074/TemplateConstructionConcept.qll @@ -0,0 +1,159 @@ +private import python +private import semmle.python.dataflow.new.DataFlow +private import semmle.python.ApiGraphs + +/** + * A data-flow node that constructs a template. + * + * Extend this class to refine existing API models. If you want to model new APIs, + * extend `TemplateConstruction::Range` instead. + */ +class TemplateConstruction extends DataFlow::Node instanceof TemplateConstruction::Range { + /** Gets the argument that specifies the template source. */ + DataFlow::Node getSourceArg() { result = super.getSourceArg() } +} + +/** Provides a class for modeling new system-command execution APIs. */ +module TemplateConstruction { + /** + * A data-flow node that constructs a template. + * + * Extend this class to model new APIs. If you want to refine existing API models, + * extend `TemplateConstruction` instead. + */ + abstract class Range extends DataFlow::Node { + /** Gets the argument that specifies the template source. */ + abstract DataFlow::Node getSourceArg(); + } +} + +// ----------------------------------------------------------------------------- +/** A call to `airspeed.Template`. */ +class AirspeedTemplateConstruction extends TemplateConstruction::Range, API::CallNode { + AirspeedTemplateConstruction() { + this = API::moduleImport("airspeed").getMember("Template").getACall() + } + + override DataFlow::Node getSourceArg() { result = this.getArg(0) } +} + +/** A call to `bottle.SimpleTemplate`. */ +class BottleSimpleTemplateConstruction extends TemplateConstruction::Range, API::CallNode { + BottleSimpleTemplateConstruction() { + this = API::moduleImport("bottle").getMember("SimpleTemplate").getACall() + } + + override DataFlow::Node getSourceArg() { result = this.getArg(0) } +} + +/** A call to `bottle.template`. */ +class BottleTemplateConstruction extends TemplateConstruction::Range, API::CallNode { + BottleTemplateConstruction() { + this = API::moduleImport("bottle").getMember("template").getACall() + } + + override DataFlow::Node getSourceArg() { result = this.getArg(0) } +} + +/** A call to `chameleon.PageTemplate`. */ +class ChameleonTemplateConstruction extends TemplateConstruction::Range, API::CallNode { + ChameleonTemplateConstruction() { + this = API::moduleImport("chameleon").getMember("PageTemplate").getACall() + } + + override DataFlow::Node getSourceArg() { result = this.getArg(0) } +} + +/** A call to `Cheetah.Template.Template`. */ +class CheetahTemplateConstruction extends TemplateConstruction::Range, API::CallNode { + CheetahTemplateConstruction() { + this = + API::moduleImport("Cheetah") + .getMember("Template") + .getMember("Template") + .getASubclass*() + .getACall() + } + + override DataFlow::Node getSourceArg() { result = this.getArg(0) } +} + +/** A call to `chevron.render`. */ +class ChevronRenderConstruction extends TemplateConstruction::Range, API::CallNode { + ChevronRenderConstruction() { this = API::moduleImport("chevron").getMember("render").getACall() } + + override DataFlow::Node getSourceArg() { result = this.getArg(0) } +} + +/** A call to `django.template.Template` */ +class DjangoTemplateConstruction extends TemplateConstruction::Range, API::CallNode { + DjangoTemplateConstruction() { + this = API::moduleImport("django").getMember("template").getMember("Template").getACall() + } + + override DataFlow::Node getSourceArg() { result = this.getArg(0) } +} + +// TODO: support django.template.engines["django"]].from_string +/** A call to `flask.render_template_string`. */ +class FlaskTemplateConstruction extends TemplateConstruction::Range, API::CallNode { + FlaskTemplateConstruction() { + this = API::moduleImport("flask").getMember("render_template_string").getACall() + } + + override DataFlow::Node getSourceArg() { result = this.getArg(0) } +} + +/** A call to `genshi.template.TextTemplate`. */ +class GenshiTextTemplateConstruction extends TemplateConstruction::Range, API::CallNode { + GenshiTextTemplateConstruction() { + this = API::moduleImport("genshi").getMember("template").getMember("TextTemplate").getACall() + } + + override DataFlow::Node getSourceArg() { result = this.getArg(0) } +} + +/** A call to `genshi.template.MarkupTemplate` */ +class GenshiMarkupTemplateConstruction extends TemplateConstruction::Range, API::CallNode { + GenshiMarkupTemplateConstruction() { + this = API::moduleImport("genshi").getMember("template").getMember("MarkupTemplate").getACall() + } + + override DataFlow::Node getSourceArg() { result = this.getArg(0) } +} + +/** A call to `jinja2.Template`. */ +class Jinja2TemplateConstruction extends TemplateConstruction::Range, API::CallNode { + Jinja2TemplateConstruction() { + this = API::moduleImport("jinja2").getMember("Template").getACall() + } + + override DataFlow::Node getSourceArg() { result = this.getArg(0) } +} + +/** A call to `jinja2.from_string`. */ +class Jinja2FromStringConstruction extends TemplateConstruction::Range, API::CallNode { + Jinja2FromStringConstruction() { + this = API::moduleImport("jinja2").getMember("from_string").getACall() + } + + override DataFlow::Node getSourceArg() { result = this.getArg(0) } +} + +/** A call to `mako.template.Template`. */ +class MakoTemplateConstruction extends TemplateConstruction::Range, API::CallNode { + MakoTemplateConstruction() { + this = API::moduleImport("mako").getMember("template").getMember("Template").getACall() + } + + override DataFlow::Node getSourceArg() { result = this.getArg(0) } +} + +/** A call to `trender.TRender`. */ +class TRenderTemplateConstruction extends TemplateConstruction::Range, API::CallNode { + TRenderTemplateConstruction() { + this = API::moduleImport("trender").getMember("TRender").getACall() + } + + override DataFlow::Node getSourceArg() { result = this.getArg(0) } +} diff --git a/python/ql/src/experimental/Security/CWE-074/TemplateInjection.ql b/python/ql/src/experimental/Security/CWE-074/TemplateInjection.ql index cbc3536ad7d..a10ad09a6ac 100644 --- a/python/ql/src/experimental/Security/CWE-074/TemplateInjection.ql +++ b/python/ql/src/experimental/Security/CWE-074/TemplateInjection.ql @@ -11,25 +11,10 @@ */ import python -import semmle.python.security.Paths -/* Sources */ -import semmle.python.web.HttpRequest -/* Sinks */ -import experimental.semmle.python.templates.Ssti -/* Flow */ -import semmle.python.security.strings.Untrusted +import TemplateInjectionQuery +import TemplateInjectionFlow::PathGraph -class TemplateInjectionConfiguration extends TaintTracking::Configuration { - TemplateInjectionConfiguration() { this = "Template injection configuration" } - - deprecated override predicate isSource(TaintTracking::Source source) { - source instanceof HttpRequestTaintSource - } - - deprecated override predicate isSink(TaintTracking::Sink sink) { sink instanceof SSTISink } -} - -from TemplateInjectionConfiguration config, TaintedPathSource src, TaintedPathSink sink -where config.hasFlowPath(src, sink) -select sink.getSink(), src, sink, "This Template depends on $@.", src.getSource(), - "a user-provided value" +from TemplateInjectionFlow::PathNode source, TemplateInjectionFlow::PathNode sink +where TemplateInjectionFlow::flowPath(source, sink) +select sink.getNode(), source, sink, "This Template depends on $@.", source.getNode(), + "user-provided value" diff --git a/python/ql/src/experimental/Security/CWE-074/TemplateInjectionCustomizations.qll b/python/ql/src/experimental/Security/CWE-074/TemplateInjectionCustomizations.qll new file mode 100644 index 00000000000..84d1a407c5f --- /dev/null +++ b/python/ql/src/experimental/Security/CWE-074/TemplateInjectionCustomizations.qll @@ -0,0 +1,51 @@ +/** + * Provides default sources, sinks and sanitizers for detecting + * "template injection" + * vulnerabilities, as well as extension points for adding your own. + */ + +private import python +private import semmle.python.dataflow.new.DataFlow +private import semmle.python.Concepts +private import semmle.python.dataflow.new.RemoteFlowSources +private import semmle.python.dataflow.new.BarrierGuards +private import TemplateConstructionConcept + +/** + * Provides default sources, sinks and sanitizers for detecting + * "template injection" + * vulnerabilities, as well as extension points for adding your own. + */ +module SqlInjection { + /** + * A data flow source for "template injection" vulnerabilities. + */ + abstract class Source extends DataFlow::Node { } + + /** + * A data flow sink for "template injection" vulnerabilities. + */ + abstract class Sink extends DataFlow::Node { } + + /** + * A sanitizer for "template injection" vulnerabilities. + */ + abstract class Sanitizer extends DataFlow::Node { } + + /** + * A source of remote user input, considered as a flow source. + */ + class RemoteFlowSourceAsSource extends Source, RemoteFlowSource { } + + /** + * A SQL statement of a SQL construction, considered as a flow sink. + */ + class TemplateConstructionAsSink extends Sink { + TemplateConstructionAsSink() { this = any(TemplateConstruction c).getSourceArg() } + } + + /** + * A comparison with a constant string, considered as a sanitizer-guard. + */ + class StringConstCompareAsSanitizerGuard extends Sanitizer, StringConstCompareBarrier { } +} diff --git a/python/ql/src/experimental/Security/CWE-074/TemplateInjectionQuery.qll b/python/ql/src/experimental/Security/CWE-074/TemplateInjectionQuery.qll new file mode 100644 index 00000000000..855dac0d80c --- /dev/null +++ b/python/ql/src/experimental/Security/CWE-074/TemplateInjectionQuery.qll @@ -0,0 +1,18 @@ +/** + * Provides a taint-tracking configuration for detecting "template injection" vulnerabilities. + */ + +private import python +import semmle.python.dataflow.new.DataFlow +import semmle.python.dataflow.new.TaintTracking +import TemplateInjectionCustomizations::SqlInjection + +module TemplateInjectionConfig implements DataFlow::ConfigSig { + predicate isSource(DataFlow::Node node) { node instanceof Source } + + predicate isSink(DataFlow::Node node) { node instanceof Sink } + + predicate isBarrierIn(DataFlow::Node node) { node instanceof Sanitizer } +} + +module TemplateInjectionFlow = TaintTracking::Global; diff --git a/python/ql/src/experimental/semmle/python/templates/Airspeed.qll b/python/ql/src/experimental/semmle/python/templates/Airspeed.qll deleted file mode 100644 index dc266ac0f82..00000000000 --- a/python/ql/src/experimental/semmle/python/templates/Airspeed.qll +++ /dev/null @@ -1,27 +0,0 @@ -/** Provides classes which model the `airspeed` package. */ - -import python -import semmle.python.web.HttpRequest -import experimental.semmle.python.templates.SSTISink - -/** returns the ClassValue representing `airspeed.Template` */ -deprecated ClassValue theAirspeedTemplateClass() { result = Value::named("airspeed.Template") } - -/** - * A sink representing the `airspeed.Template` class instantiation argument. - * - * import airspeed - * temp = airspeed.Template(`"sink"`) - */ -deprecated class AirspeedTemplateSink extends SSTISink { - override string toString() { result = "argument to airspeed.Template()" } - - AirspeedTemplateSink() { - exists(CallNode call | - call.getFunction().pointsTo(theAirspeedTemplateClass()) and - call.getArg(0) = this - ) - } - - override predicate sinks(TaintKind kind) { kind instanceof ExternalStringKind } -} diff --git a/python/ql/src/experimental/semmle/python/templates/Bottle.qll b/python/ql/src/experimental/semmle/python/templates/Bottle.qll deleted file mode 100644 index 1f5bd2bba85..00000000000 --- a/python/ql/src/experimental/semmle/python/templates/Bottle.qll +++ /dev/null @@ -1,48 +0,0 @@ -/** Provides classes which model the `bottle` package. */ - -import python -import semmle.python.web.HttpRequest -import experimental.semmle.python.templates.SSTISink - -/** returns the ClassValue representing `bottle.SimpleTemplate` */ -deprecated ClassValue theBottleSimpleTemplateClass() { - result = Value::named("bottle.SimpleTemplate") -} - -/** - * A sink representing the `bottle.SimpleTemplate` class instantiation argument. - * - * from bottle import SimpleTemplate - * template = SimpleTemplate(`sink`) - */ -deprecated class BottleSimpleTemplateSink extends SSTISink { - override string toString() { result = "argument to bottle.SimpleTemplate()" } - - BottleSimpleTemplateSink() { - exists(CallNode call | - call.getFunction().pointsTo(theBottleSimpleTemplateClass()) and - call.getArg(0) = this - ) - } - - override predicate sinks(TaintKind kind) { kind instanceof ExternalStringKind } -} - -/** - * A sink representing the `bottle.template` function call argument. - * - * from bottle import template - * tmp = template(`sink`) - */ -deprecated class BottleTemplateSink extends SSTISink { - override string toString() { result = "argument to bottle.template()" } - - BottleTemplateSink() { - exists(CallNode call | - call.getFunction() = theBottleModule().attr("template").getAReference() and - call.getArg(0) = this - ) - } - - override predicate sinks(TaintKind kind) { kind instanceof ExternalStringKind } -} diff --git a/python/ql/src/experimental/semmle/python/templates/Chameleon.qll b/python/ql/src/experimental/semmle/python/templates/Chameleon.qll deleted file mode 100644 index f094dda97b5..00000000000 --- a/python/ql/src/experimental/semmle/python/templates/Chameleon.qll +++ /dev/null @@ -1,29 +0,0 @@ -/** Provides classes which model the `Chameleon` package. */ - -import python -import semmle.python.web.HttpRequest -import experimental.semmle.python.templates.SSTISink - -/** returns the ClassValue representing `chameleon.PageTemplate` */ -deprecated ClassValue theChameleonPageTemplateClass() { - result = Value::named("chameleon.PageTemplate") -} - -/** - * A sink representing the `chameleon.PageTemplate` class instantiation argument. - * - * from chameleon import PageTemplate - * template = PageTemplate(`sink`) - */ -deprecated class ChameleonTemplateSink extends SSTISink { - override string toString() { result = "argument to Chameleon.PageTemplate()" } - - ChameleonTemplateSink() { - exists(CallNode call | - call.getFunction().pointsTo(theChameleonPageTemplateClass()) and - call.getArg(0) = this - ) - } - - override predicate sinks(TaintKind kind) { kind instanceof ExternalStringKind } -} diff --git a/python/ql/src/experimental/semmle/python/templates/Cheetah.qll b/python/ql/src/experimental/semmle/python/templates/Cheetah.qll deleted file mode 100644 index 9812fdb7c88..00000000000 --- a/python/ql/src/experimental/semmle/python/templates/Cheetah.qll +++ /dev/null @@ -1,39 +0,0 @@ -/** Provides classes which model the `Cheetah3` package. */ - -import python -import semmle.python.web.HttpRequest -import experimental.semmle.python.templates.SSTISink - -/** returns the ClassValue representing `Cheetah.Template.Template` */ -deprecated ClassValue theCheetahTemplateClass() { - result = Value::named("Cheetah.Template.Template") -} - -/** - * A sink representing the instantiation argument of any class which derives from - * the `Cheetah.Template.Template` class . - * - * from Cheetah.Template import Template - * class Template3(Template): - * title = 'Hello World Example!' - * contents = 'Hello World!' - * t3 = Template3("sink") - * - * This will also detect cases of the following type : - * - * from Cheetah.Template import Template - * t3 = Template("sink") - */ -deprecated class CheetahTemplateInstantiationSink extends SSTISink { - override string toString() { result = "argument to Cheetah.Template.Template()" } - - CheetahTemplateInstantiationSink() { - exists(CallNode call, ClassValue cv | - cv.getASuperType() = theCheetahTemplateClass() and - call.getFunction().pointsTo(cv) and - call.getArg(0) = this - ) - } - - override predicate sinks(TaintKind kind) { kind instanceof ExternalStringKind } -} diff --git a/python/ql/src/experimental/semmle/python/templates/Chevron.qll b/python/ql/src/experimental/semmle/python/templates/Chevron.qll deleted file mode 100644 index cc93016891c..00000000000 --- a/python/ql/src/experimental/semmle/python/templates/Chevron.qll +++ /dev/null @@ -1,36 +0,0 @@ -/** Provides classes which model the `chevron` package. */ - -import python -import semmle.python.web.HttpRequest -import experimental.semmle.python.templates.SSTISink - -/** returns the Value representing `chevron.render` function */ -deprecated Value theChevronRenderFunc() { result = Value::named("chevron.render") } - -/** - * A sink representing the `chevron.render` function call argument. - * - * import chevron - * tmp = chevron.render(`sink`,{ 'key' : 'value' }) - */ -deprecated class ChevronRenderSink extends SSTISink { - override string toString() { result = "argument to chevron.render()" } - - ChevronRenderSink() { - exists(CallNode call | - call.getFunction() = theChevronRenderFunc().getAReference() and - call.getArg(0) = this - ) - // TODO: this should also detect : - // import chevron - // args = { - // 'template': 'sink', - // 'data': { - // 'mustache': 'World' - // } - // } - // chevron.render(**args) - } - - override predicate sinks(TaintKind kind) { kind instanceof ExternalStringKind } -} diff --git a/python/ql/src/experimental/semmle/python/templates/DjangoTemplate.qll b/python/ql/src/experimental/semmle/python/templates/DjangoTemplate.qll deleted file mode 100644 index 1089ab872ec..00000000000 --- a/python/ql/src/experimental/semmle/python/templates/DjangoTemplate.qll +++ /dev/null @@ -1,35 +0,0 @@ -/** Provides classes which model the `DjangoTemplate` package. */ - -import python -import semmle.python.web.HttpRequest -import experimental.semmle.python.templates.SSTISink - -deprecated ClassValue theDjangoTemplateClass() { result = Value::named("django.template.Template") } - -/** - * A sink representing `django.template.Template` class instantiation argument. - * - * from django.template import Template - * template = Template(`sink`) - */ -deprecated class DjangoTemplateTemplateSink extends SSTISink { - override string toString() { result = "argument to Django.template()" } - - DjangoTemplateTemplateSink() { - exists(CallNode call | - call.getFunction().pointsTo(theDjangoTemplateClass()) and - call.getArg(0) = this - ) - } - - override predicate sinks(TaintKind kind) { kind instanceof ExternalStringKind } -} -// TODO (intentionally commented out QLDoc, since qlformat will delete those lines otherwise) -// /** -// * Sinks representing the django.template.Template class instantiation. -// * -// * from django.template import engines -// * -// * django_engine = engines["django"] -// * template = django_engine.from_string(`sink`) -// */ diff --git a/python/ql/src/experimental/semmle/python/templates/FlaskTemplate.qll b/python/ql/src/experimental/semmle/python/templates/FlaskTemplate.qll deleted file mode 100644 index c0f3c90235d..00000000000 --- a/python/ql/src/experimental/semmle/python/templates/FlaskTemplate.qll +++ /dev/null @@ -1,28 +0,0 @@ -/** Provides classes which model templates in the`flask` package. */ - -import python -import semmle.python.web.HttpRequest -import experimental.semmle.python.templates.SSTISink - -deprecated Value theFlaskRenderTemplateClass() { - result = Value::named("flask.render_template_string") -} - -/** - * A sink representing `flask.render_template_string` function call argument. - * - * from flask import render_template_string - * render_template_string(`sink`) - */ -deprecated class FlaskTemplateSink extends SSTISink { - override string toString() { result = "argument to flask.render_template_string()" } - - FlaskTemplateSink() { - exists(CallNode call | - call.getFunction().pointsTo(theFlaskRenderTemplateClass()) and - call.getArg(0) = this - ) - } - - override predicate sinks(TaintKind kind) { kind instanceof ExternalStringKind } -} diff --git a/python/ql/src/experimental/semmle/python/templates/Genshi.qll b/python/ql/src/experimental/semmle/python/templates/Genshi.qll deleted file mode 100644 index c808d7c60f8..00000000000 --- a/python/ql/src/experimental/semmle/python/templates/Genshi.qll +++ /dev/null @@ -1,53 +0,0 @@ -/** Provides classes which model the `Genshi` package. */ - -import python -import semmle.python.web.HttpRequest -import experimental.semmle.python.templates.SSTISink - -/** returns the ClassValue representing `Genshi.template.TextTemplate` */ -deprecated ClassValue theGenshiTextTemplateClass() { - result = Value::named("genshi.template.TextTemplate") -} - -/** returns the ClassValue representing `Genshi.template.MarkupTemplate` */ -deprecated ClassValue theGenshiMarkupTemplateClass() { - result = Value::named("genshi.template.MarkupTemplate") -} - -/** - * A sink representing the `genshi.template.TextTemplate` class instantiation argument. - * - * from genshi.template import TextTemplate - * tmpl = TextTemplate('sink') - */ -deprecated class GenshiTextTemplateSink extends SSTISink { - override string toString() { result = "argument to genshi.template.TextTemplate()" } - - GenshiTextTemplateSink() { - exists(CallNode call | - call.getFunction().pointsTo(theGenshiTextTemplateClass()) and - call.getArg(0) = this - ) - } - - override predicate sinks(TaintKind kind) { kind instanceof ExternalStringKind } -} - -/** - * A sink representing the `genshi.template.MarkupTemplate` class instantiation argument. - * - * from genshi.template import MarkupTemplate - * tmpl = MarkupTemplate('sink') - */ -deprecated class GenshiMarkupTemplateSink extends SSTISink { - override string toString() { result = "argument to genshi.template.MarkupTemplate()" } - - GenshiMarkupTemplateSink() { - exists(CallNode call | - call.getFunction().pointsTo(theGenshiMarkupTemplateClass()) and - call.getArg(0) = this - ) - } - - override predicate sinks(TaintKind kind) { kind instanceof ExternalStringKind } -} diff --git a/python/ql/src/experimental/semmle/python/templates/Jinja.qll b/python/ql/src/experimental/semmle/python/templates/Jinja.qll deleted file mode 100644 index 44bc103cf04..00000000000 --- a/python/ql/src/experimental/semmle/python/templates/Jinja.qll +++ /dev/null @@ -1,49 +0,0 @@ -/** Provides classes which model the `Jinja2` package. */ - -import python -import semmle.python.web.HttpRequest -import experimental.semmle.python.templates.SSTISink - -/** returns the ClassValue representing `jinja2.Template` */ -deprecated ClassValue theJinja2TemplateClass() { result = Value::named("jinja2.Template") } - -/** returns the ClassValue representing `jinja2.Template` */ -deprecated Value theJinja2FromStringValue() { result = Value::named("jinja2.from_string") } - -/** - * A sink representing the `jinja2.Template` class instantiation argument. - * - * from jinja2 import Template - * template = Template(`sink`) - */ -deprecated class Jinja2TemplateSink extends SSTISink { - override string toString() { result = "argument to jinja2.Template()" } - - Jinja2TemplateSink() { - exists(CallNode call | - call.getFunction().pointsTo(theJinja2TemplateClass()) and - call.getArg(0) = this - ) - } - - override predicate sinks(TaintKind kind) { kind instanceof ExternalStringKind } -} - -/** - * A sink representing the `jinja2.from_string` function call argument. - * - * from jinja2 import from_string - * template = from_string(`sink`) - */ -deprecated class Jinja2FromStringSink extends SSTISink { - override string toString() { result = "argument to jinja2.from_string()" } - - Jinja2FromStringSink() { - exists(CallNode call | - call.getFunction().pointsTo(theJinja2FromStringValue()) and - call.getArg(0) = this - ) - } - - override predicate sinks(TaintKind kind) { kind instanceof ExternalStringKind } -} diff --git a/python/ql/src/experimental/semmle/python/templates/Mako.qll b/python/ql/src/experimental/semmle/python/templates/Mako.qll deleted file mode 100644 index b8634b3001a..00000000000 --- a/python/ql/src/experimental/semmle/python/templates/Mako.qll +++ /dev/null @@ -1,27 +0,0 @@ -/** Provides classes which model the `Mako` package. */ - -import python -import semmle.python.web.HttpRequest -import experimental.semmle.python.templates.SSTISink - -/** returns the ClassValue representing `mako.template.Template` */ -deprecated ClassValue theMakoTemplateClass() { result = Value::named("mako.template.Template") } - -/** - * A sink representing the `mako.template.Template` class instantiation argument. - * - * from mako.template import Template - * mytemplate = Template("hello world!") - */ -deprecated class MakoTemplateSink extends SSTISink { - override string toString() { result = "argument to mako.template.Template()" } - - MakoTemplateSink() { - exists(CallNode call | - call.getFunction().pointsTo(theMakoTemplateClass()) and - call.getArg(0) = this - ) - } - - override predicate sinks(TaintKind kind) { kind instanceof ExternalStringKind } -} diff --git a/python/ql/src/experimental/semmle/python/templates/SSTISink.qll b/python/ql/src/experimental/semmle/python/templates/SSTISink.qll deleted file mode 100644 index 1a68fe17b68..00000000000 --- a/python/ql/src/experimental/semmle/python/templates/SSTISink.qll +++ /dev/null @@ -1,7 +0,0 @@ -import semmle.python.dataflow.TaintTracking - -/** - * A generic taint sink that is vulnerable to template inclusions. - * The `temp` in `jinja2.Template(temp)` and similar. - */ -abstract deprecated class SSTISink extends TaintSink { } diff --git a/python/ql/src/experimental/semmle/python/templates/Ssti.qll b/python/ql/src/experimental/semmle/python/templates/Ssti.qll deleted file mode 100644 index eb4f8d0ec2f..00000000000 --- a/python/ql/src/experimental/semmle/python/templates/Ssti.qll +++ /dev/null @@ -1,13 +0,0 @@ -/** Imports all files which model potential SSTI sinks */ - -import experimental.semmle.python.templates.Airspeed -import experimental.semmle.python.templates.Bottle -import experimental.semmle.python.templates.Chameleon -import experimental.semmle.python.templates.Cheetah -import experimental.semmle.python.templates.Chevron -import experimental.semmle.python.templates.DjangoTemplate -import experimental.semmle.python.templates.FlaskTemplate -import experimental.semmle.python.templates.Genshi -import experimental.semmle.python.templates.Jinja -import experimental.semmle.python.templates.Mako -import experimental.semmle.python.templates.TRender diff --git a/python/ql/src/experimental/semmle/python/templates/TRender.qll b/python/ql/src/experimental/semmle/python/templates/TRender.qll deleted file mode 100644 index 8d5431ad9e0..00000000000 --- a/python/ql/src/experimental/semmle/python/templates/TRender.qll +++ /dev/null @@ -1,27 +0,0 @@ -/** Provides classes which model the `TRender` package. */ - -import python -import semmle.python.web.HttpRequest -import experimental.semmle.python.templates.SSTISink - -/** returns the ClassValue representing `trender.TRender` */ -deprecated ClassValue theTRenderTemplateClass() { result = Value::named("trender.TRender") } - -/** - * A sink representing the `trender.TRender` class instantiation argument. - * - * from trender import TRender - * template = TRender(`sink`) - */ -deprecated class TRenderTemplateSink extends SSTISink { - override string toString() { result = "argument to trender.TRender()" } - - TRenderTemplateSink() { - exists(CallNode call | - call.getFunction().pointsTo(theTRenderTemplateClass()) and - call.getArg(0) = this - ) - } - - override predicate sinks(TaintKind kind) { kind instanceof ExternalStringKind } -} diff --git a/python/ql/test/experimental/query-tests/Security/CWE-074/TemplateInjection.expected b/python/ql/test/experimental/query-tests/Security/CWE-074/TemplateInjection.expected index 058a53bdf91..188aec4c024 100644 --- a/python/ql/test/experimental/query-tests/Security/CWE-074/TemplateInjection.expected +++ b/python/ql/test/experimental/query-tests/Security/CWE-074/TemplateInjection.expected @@ -1,60 +1,88 @@ edges -| AirspeedSsti.py:10:16:10:27 | dict of externally controlled string | AirspeedSsti.py:10:16:10:43 | externally controlled string | -| AirspeedSsti.py:10:16:10:27 | dict of externally controlled string | AirspeedSsti.py:10:16:10:43 | externally controlled string | -| AirspeedSsti.py:10:16:10:43 | externally controlled string | AirspeedSsti.py:11:30:11:37 | externally controlled string | -| AirspeedSsti.py:10:16:10:43 | externally controlled string | AirspeedSsti.py:11:30:11:37 | externally controlled string | -| ChevronSsti.py:10:16:10:27 | dict of externally controlled string | ChevronSsti.py:10:16:10:43 | externally controlled string | -| ChevronSsti.py:10:16:10:27 | dict of externally controlled string | ChevronSsti.py:10:16:10:43 | externally controlled string | -| ChevronSsti.py:10:16:10:43 | externally controlled string | ChevronSsti.py:11:27:11:34 | externally controlled string | -| ChevronSsti.py:10:16:10:43 | externally controlled string | ChevronSsti.py:11:27:11:34 | externally controlled string | -| DjangoTemplates.py:6:8:6:14 | django.request.HttpRequest | DjangoTemplates.py:8:16:8:22 | django.request.HttpRequest | -| DjangoTemplates.py:6:8:6:14 | django.request.HttpRequest | DjangoTemplates.py:8:16:8:22 | django.request.HttpRequest | -| DjangoTemplates.py:8:16:8:22 | django.request.HttpRequest | DjangoTemplates.py:8:16:8:26 | django.http.request.QueryDict | -| DjangoTemplates.py:8:16:8:22 | django.request.HttpRequest | DjangoTemplates.py:8:16:8:26 | django.http.request.QueryDict | -| DjangoTemplates.py:8:16:8:26 | django.http.request.QueryDict | DjangoTemplates.py:8:16:8:38 | externally controlled string | -| DjangoTemplates.py:8:16:8:26 | django.http.request.QueryDict | DjangoTemplates.py:8:16:8:38 | externally controlled string | -| DjangoTemplates.py:8:16:8:38 | externally controlled string | DjangoTemplates.py:9:18:9:25 | externally controlled string | -| DjangoTemplates.py:8:16:8:38 | externally controlled string | DjangoTemplates.py:9:18:9:25 | externally controlled string | -| FlaskTemplate.py:17:41:17:52 | dict of externally controlled string | FlaskTemplate.py:17:41:17:68 | externally controlled string | -| FlaskTemplate.py:17:41:17:52 | dict of externally controlled string | FlaskTemplate.py:17:41:17:68 | externally controlled string | -| JinjaSsti.py:7:7:7:13 | django.request.HttpRequest | JinjaSsti.py:9:16:9:22 | django.request.HttpRequest | -| JinjaSsti.py:7:7:7:13 | django.request.HttpRequest | JinjaSsti.py:9:16:9:22 | django.request.HttpRequest | -| JinjaSsti.py:9:16:9:22 | django.request.HttpRequest | JinjaSsti.py:9:16:9:26 | django.http.request.QueryDict | -| JinjaSsti.py:9:16:9:22 | django.request.HttpRequest | JinjaSsti.py:9:16:9:26 | django.http.request.QueryDict | -| JinjaSsti.py:9:16:9:26 | django.http.request.QueryDict | JinjaSsti.py:9:16:9:38 | externally controlled string | -| JinjaSsti.py:9:16:9:26 | django.http.request.QueryDict | JinjaSsti.py:9:16:9:38 | externally controlled string | -| JinjaSsti.py:9:16:9:38 | externally controlled string | JinjaSsti.py:10:25:10:32 | externally controlled string | -| JinjaSsti.py:9:16:9:38 | externally controlled string | JinjaSsti.py:10:25:10:32 | externally controlled string | -| JinjaSsti.py:16:7:16:13 | django.request.HttpRequest | JinjaSsti.py:19:16:19:22 | django.request.HttpRequest | -| JinjaSsti.py:16:7:16:13 | django.request.HttpRequest | JinjaSsti.py:19:16:19:22 | django.request.HttpRequest | -| JinjaSsti.py:19:16:19:22 | django.request.HttpRequest | JinjaSsti.py:19:16:19:26 | django.http.request.QueryDict | -| JinjaSsti.py:19:16:19:22 | django.request.HttpRequest | JinjaSsti.py:19:16:19:26 | django.http.request.QueryDict | -| JinjaSsti.py:19:16:19:26 | django.http.request.QueryDict | JinjaSsti.py:19:16:19:38 | externally controlled string | -| JinjaSsti.py:19:16:19:26 | django.http.request.QueryDict | JinjaSsti.py:19:16:19:38 | externally controlled string | -| JinjaSsti.py:19:16:19:38 | externally controlled string | JinjaSsti.py:20:28:20:35 | externally controlled string | -| JinjaSsti.py:19:16:19:38 | externally controlled string | JinjaSsti.py:20:28:20:35 | externally controlled string | -| MakoSsti.py:6:10:6:16 | django.request.HttpRequest | MakoSsti.py:8:16:8:22 | django.request.HttpRequest | -| MakoSsti.py:6:10:6:16 | django.request.HttpRequest | MakoSsti.py:8:16:8:22 | django.request.HttpRequest | -| MakoSsti.py:8:16:8:22 | django.request.HttpRequest | MakoSsti.py:8:16:8:26 | django.http.request.QueryDict | -| MakoSsti.py:8:16:8:22 | django.request.HttpRequest | MakoSsti.py:8:16:8:26 | django.http.request.QueryDict | -| MakoSsti.py:8:16:8:26 | django.http.request.QueryDict | MakoSsti.py:8:16:8:38 | externally controlled string | -| MakoSsti.py:8:16:8:26 | django.http.request.QueryDict | MakoSsti.py:8:16:8:38 | externally controlled string | -| MakoSsti.py:8:16:8:38 | externally controlled string | MakoSsti.py:9:27:9:34 | externally controlled string | -| MakoSsti.py:8:16:8:38 | externally controlled string | MakoSsti.py:9:27:9:34 | externally controlled string | -| TRender.py:5:13:5:19 | django.request.HttpRequest | TRender.py:6:16:6:22 | django.request.HttpRequest | -| TRender.py:5:13:5:19 | django.request.HttpRequest | TRender.py:6:16:6:22 | django.request.HttpRequest | -| TRender.py:6:16:6:22 | django.request.HttpRequest | TRender.py:6:16:6:26 | django.http.request.QueryDict | -| TRender.py:6:16:6:22 | django.request.HttpRequest | TRender.py:6:16:6:26 | django.http.request.QueryDict | -| TRender.py:6:16:6:26 | django.http.request.QueryDict | TRender.py:6:16:6:38 | externally controlled string | -| TRender.py:6:16:6:26 | django.http.request.QueryDict | TRender.py:6:16:6:38 | externally controlled string | -| TRender.py:6:16:6:38 | externally controlled string | TRender.py:7:24:7:31 | externally controlled string | -| TRender.py:6:16:6:38 | externally controlled string | TRender.py:7:24:7:31 | externally controlled string | +| AirspeedSsti.py:2:26:2:32 | ControlFlowNode for ImportMember | AirspeedSsti.py:2:26:2:32 | GSSA Variable request | +| AirspeedSsti.py:2:26:2:32 | GSSA Variable request | AirspeedSsti.py:10:16:10:22 | ControlFlowNode for request | +| AirspeedSsti.py:10:16:10:22 | ControlFlowNode for request | AirspeedSsti.py:10:16:10:27 | ControlFlowNode for Attribute | +| AirspeedSsti.py:10:16:10:27 | ControlFlowNode for Attribute | AirspeedSsti.py:10:16:10:43 | ControlFlowNode for Attribute() | +| AirspeedSsti.py:10:16:10:43 | ControlFlowNode for Attribute() | AirspeedSsti.py:11:30:11:37 | ControlFlowNode for template | +| ChevronSsti.py:1:26:1:32 | ControlFlowNode for ImportMember | ChevronSsti.py:1:26:1:32 | GSSA Variable request | +| ChevronSsti.py:1:26:1:32 | GSSA Variable request | ChevronSsti.py:10:16:10:22 | ControlFlowNode for request | +| ChevronSsti.py:10:16:10:22 | ControlFlowNode for request | ChevronSsti.py:10:16:10:27 | ControlFlowNode for Attribute | +| ChevronSsti.py:10:16:10:27 | ControlFlowNode for Attribute | ChevronSsti.py:10:16:10:43 | ControlFlowNode for Attribute() | +| ChevronSsti.py:10:16:10:43 | ControlFlowNode for Attribute() | ChevronSsti.py:11:27:11:34 | ControlFlowNode for template | +| DjangoTemplates.py:6:8:6:14 | ControlFlowNode for request | DjangoTemplates.py:8:16:8:26 | ControlFlowNode for Attribute | +| DjangoTemplates.py:8:16:8:26 | ControlFlowNode for Attribute | DjangoTemplates.py:8:16:8:38 | ControlFlowNode for Subscript | +| DjangoTemplates.py:8:16:8:38 | ControlFlowNode for Subscript | DjangoTemplates.py:9:18:9:25 | ControlFlowNode for template | +| FlaskTemplate.py:1:26:1:32 | ControlFlowNode for ImportMember | FlaskTemplate.py:1:26:1:32 | GSSA Variable request | +| FlaskTemplate.py:1:26:1:32 | GSSA Variable request | FlaskTemplate.py:10:8:10:14 | ControlFlowNode for request | +| FlaskTemplate.py:1:26:1:32 | GSSA Variable request | FlaskTemplate.py:11:39:11:45 | ControlFlowNode for request | +| FlaskTemplate.py:1:26:1:32 | GSSA Variable request | FlaskTemplate.py:17:41:17:47 | ControlFlowNode for request | +| FlaskTemplate.py:10:8:10:14 | ControlFlowNode for request | FlaskTemplate.py:11:39:11:50 | ControlFlowNode for Attribute | +| FlaskTemplate.py:11:39:11:45 | ControlFlowNode for request | FlaskTemplate.py:11:39:11:50 | ControlFlowNode for Attribute | +| FlaskTemplate.py:11:39:11:50 | ControlFlowNode for Attribute | FlaskTemplate.py:11:39:11:66 | ControlFlowNode for Attribute() | +| FlaskTemplate.py:17:41:17:47 | ControlFlowNode for request | FlaskTemplate.py:17:41:17:52 | ControlFlowNode for Attribute | +| FlaskTemplate.py:17:41:17:52 | ControlFlowNode for Attribute | FlaskTemplate.py:17:41:17:68 | ControlFlowNode for Attribute() | +| JinjaSsti.py:7:7:7:13 | ControlFlowNode for request | JinjaSsti.py:9:16:9:26 | ControlFlowNode for Attribute | +| JinjaSsti.py:9:16:9:26 | ControlFlowNode for Attribute | JinjaSsti.py:9:16:9:38 | ControlFlowNode for Subscript | +| JinjaSsti.py:9:16:9:38 | ControlFlowNode for Subscript | JinjaSsti.py:10:25:10:32 | ControlFlowNode for template | +| JinjaSsti.py:16:7:16:13 | ControlFlowNode for request | JinjaSsti.py:19:16:19:26 | ControlFlowNode for Attribute | +| JinjaSsti.py:19:16:19:26 | ControlFlowNode for Attribute | JinjaSsti.py:19:16:19:38 | ControlFlowNode for Subscript | +| JinjaSsti.py:19:16:19:38 | ControlFlowNode for Subscript | JinjaSsti.py:20:28:20:35 | ControlFlowNode for template | +| MakoSsti.py:6:10:6:16 | ControlFlowNode for request | MakoSsti.py:8:16:8:26 | ControlFlowNode for Attribute | +| MakoSsti.py:8:16:8:26 | ControlFlowNode for Attribute | MakoSsti.py:8:16:8:38 | ControlFlowNode for Subscript | +| MakoSsti.py:8:16:8:38 | ControlFlowNode for Subscript | MakoSsti.py:9:27:9:34 | ControlFlowNode for template | +| TRender.py:5:13:5:19 | ControlFlowNode for request | TRender.py:6:16:6:26 | ControlFlowNode for Attribute | +| TRender.py:6:16:6:26 | ControlFlowNode for Attribute | TRender.py:6:16:6:38 | ControlFlowNode for Subscript | +| TRender.py:6:16:6:38 | ControlFlowNode for Subscript | TRender.py:7:24:7:31 | ControlFlowNode for template | +nodes +| AirspeedSsti.py:2:26:2:32 | ControlFlowNode for ImportMember | semmle.label | ControlFlowNode for ImportMember | +| AirspeedSsti.py:2:26:2:32 | GSSA Variable request | semmle.label | GSSA Variable request | +| AirspeedSsti.py:10:16:10:22 | ControlFlowNode for request | semmle.label | ControlFlowNode for request | +| AirspeedSsti.py:10:16:10:27 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute | +| AirspeedSsti.py:10:16:10:43 | ControlFlowNode for Attribute() | semmle.label | ControlFlowNode for Attribute() | +| AirspeedSsti.py:11:30:11:37 | ControlFlowNode for template | semmle.label | ControlFlowNode for template | +| ChevronSsti.py:1:26:1:32 | ControlFlowNode for ImportMember | semmle.label | ControlFlowNode for ImportMember | +| ChevronSsti.py:1:26:1:32 | GSSA Variable request | semmle.label | GSSA Variable request | +| ChevronSsti.py:10:16:10:22 | ControlFlowNode for request | semmle.label | ControlFlowNode for request | +| ChevronSsti.py:10:16:10:27 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute | +| ChevronSsti.py:10:16:10:43 | ControlFlowNode for Attribute() | semmle.label | ControlFlowNode for Attribute() | +| ChevronSsti.py:11:27:11:34 | ControlFlowNode for template | semmle.label | ControlFlowNode for template | +| DjangoTemplates.py:6:8:6:14 | ControlFlowNode for request | semmle.label | ControlFlowNode for request | +| DjangoTemplates.py:8:16:8:26 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute | +| DjangoTemplates.py:8:16:8:38 | ControlFlowNode for Subscript | semmle.label | ControlFlowNode for Subscript | +| DjangoTemplates.py:9:18:9:25 | ControlFlowNode for template | semmle.label | ControlFlowNode for template | +| FlaskTemplate.py:1:26:1:32 | ControlFlowNode for ImportMember | semmle.label | ControlFlowNode for ImportMember | +| FlaskTemplate.py:1:26:1:32 | GSSA Variable request | semmle.label | GSSA Variable request | +| FlaskTemplate.py:10:8:10:14 | ControlFlowNode for request | semmle.label | ControlFlowNode for request | +| FlaskTemplate.py:11:39:11:45 | ControlFlowNode for request | semmle.label | ControlFlowNode for request | +| FlaskTemplate.py:11:39:11:50 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute | +| FlaskTemplate.py:11:39:11:66 | ControlFlowNode for Attribute() | semmle.label | ControlFlowNode for Attribute() | +| FlaskTemplate.py:17:41:17:47 | ControlFlowNode for request | semmle.label | ControlFlowNode for request | +| FlaskTemplate.py:17:41:17:52 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute | +| FlaskTemplate.py:17:41:17:68 | ControlFlowNode for Attribute() | semmle.label | ControlFlowNode for Attribute() | +| JinjaSsti.py:7:7:7:13 | ControlFlowNode for request | semmle.label | ControlFlowNode for request | +| JinjaSsti.py:9:16:9:26 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute | +| JinjaSsti.py:9:16:9:38 | ControlFlowNode for Subscript | semmle.label | ControlFlowNode for Subscript | +| JinjaSsti.py:10:25:10:32 | ControlFlowNode for template | semmle.label | ControlFlowNode for template | +| JinjaSsti.py:16:7:16:13 | ControlFlowNode for request | semmle.label | ControlFlowNode for request | +| JinjaSsti.py:19:16:19:26 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute | +| JinjaSsti.py:19:16:19:38 | ControlFlowNode for Subscript | semmle.label | ControlFlowNode for Subscript | +| JinjaSsti.py:20:28:20:35 | ControlFlowNode for template | semmle.label | ControlFlowNode for template | +| MakoSsti.py:6:10:6:16 | ControlFlowNode for request | semmle.label | ControlFlowNode for request | +| MakoSsti.py:8:16:8:26 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute | +| MakoSsti.py:8:16:8:38 | ControlFlowNode for Subscript | semmle.label | ControlFlowNode for Subscript | +| MakoSsti.py:9:27:9:34 | ControlFlowNode for template | semmle.label | ControlFlowNode for template | +| TRender.py:5:13:5:19 | ControlFlowNode for request | semmle.label | ControlFlowNode for request | +| TRender.py:6:16:6:26 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute | +| TRender.py:6:16:6:38 | ControlFlowNode for Subscript | semmle.label | ControlFlowNode for Subscript | +| TRender.py:7:24:7:31 | ControlFlowNode for template | semmle.label | ControlFlowNode for template | +subpaths #select -| AirspeedSsti.py:11:30:11:37 | template | AirspeedSsti.py:10:16:10:27 | dict of externally controlled string | AirspeedSsti.py:11:30:11:37 | externally controlled string | This Template depends on $@. | AirspeedSsti.py:10:16:10:27 | Attribute | a user-provided value | -| ChevronSsti.py:11:27:11:34 | template | ChevronSsti.py:10:16:10:27 | dict of externally controlled string | ChevronSsti.py:11:27:11:34 | externally controlled string | This Template depends on $@. | ChevronSsti.py:10:16:10:27 | Attribute | a user-provided value | -| DjangoTemplates.py:9:18:9:25 | template | DjangoTemplates.py:6:8:6:14 | django.request.HttpRequest | DjangoTemplates.py:9:18:9:25 | externally controlled string | This Template depends on $@. | DjangoTemplates.py:6:8:6:14 | request | a user-provided value | -| FlaskTemplate.py:17:41:17:68 | Attribute() | FlaskTemplate.py:17:41:17:52 | dict of externally controlled string | FlaskTemplate.py:17:41:17:68 | externally controlled string | This Template depends on $@. | FlaskTemplate.py:17:41:17:52 | Attribute | a user-provided value | -| JinjaSsti.py:10:25:10:32 | template | JinjaSsti.py:7:7:7:13 | django.request.HttpRequest | JinjaSsti.py:10:25:10:32 | externally controlled string | This Template depends on $@. | JinjaSsti.py:7:7:7:13 | request | a user-provided value | -| JinjaSsti.py:20:28:20:35 | template | JinjaSsti.py:16:7:16:13 | django.request.HttpRequest | JinjaSsti.py:20:28:20:35 | externally controlled string | This Template depends on $@. | JinjaSsti.py:16:7:16:13 | request | a user-provided value | -| MakoSsti.py:9:27:9:34 | template | MakoSsti.py:6:10:6:16 | django.request.HttpRequest | MakoSsti.py:9:27:9:34 | externally controlled string | This Template depends on $@. | MakoSsti.py:6:10:6:16 | request | a user-provided value | -| TRender.py:7:24:7:31 | template | TRender.py:5:13:5:19 | django.request.HttpRequest | TRender.py:7:24:7:31 | externally controlled string | This Template depends on $@. | TRender.py:5:13:5:19 | request | a user-provided value | +| AirspeedSsti.py:11:30:11:37 | ControlFlowNode for template | AirspeedSsti.py:2:26:2:32 | ControlFlowNode for ImportMember | AirspeedSsti.py:11:30:11:37 | ControlFlowNode for template | This Template depends on $@. | AirspeedSsti.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | +| ChevronSsti.py:11:27:11:34 | ControlFlowNode for template | ChevronSsti.py:1:26:1:32 | ControlFlowNode for ImportMember | ChevronSsti.py:11:27:11:34 | ControlFlowNode for template | This Template depends on $@. | ChevronSsti.py:1:26:1:32 | ControlFlowNode for ImportMember | user-provided value | +| DjangoTemplates.py:9:18:9:25 | ControlFlowNode for template | DjangoTemplates.py:6:8:6:14 | ControlFlowNode for request | DjangoTemplates.py:9:18:9:25 | ControlFlowNode for template | This Template depends on $@. | DjangoTemplates.py:6:8:6:14 | ControlFlowNode for request | user-provided value | +| FlaskTemplate.py:11:39:11:66 | ControlFlowNode for Attribute() | FlaskTemplate.py:1:26:1:32 | ControlFlowNode for ImportMember | FlaskTemplate.py:11:39:11:66 | ControlFlowNode for Attribute() | This Template depends on $@. | FlaskTemplate.py:1:26:1:32 | ControlFlowNode for ImportMember | user-provided value | +| FlaskTemplate.py:17:41:17:68 | ControlFlowNode for Attribute() | FlaskTemplate.py:1:26:1:32 | ControlFlowNode for ImportMember | FlaskTemplate.py:17:41:17:68 | ControlFlowNode for Attribute() | This Template depends on $@. | FlaskTemplate.py:1:26:1:32 | ControlFlowNode for ImportMember | user-provided value | +| JinjaSsti.py:10:25:10:32 | ControlFlowNode for template | JinjaSsti.py:7:7:7:13 | ControlFlowNode for request | JinjaSsti.py:10:25:10:32 | ControlFlowNode for template | This Template depends on $@. | JinjaSsti.py:7:7:7:13 | ControlFlowNode for request | user-provided value | +| JinjaSsti.py:20:28:20:35 | ControlFlowNode for template | JinjaSsti.py:16:7:16:13 | ControlFlowNode for request | JinjaSsti.py:20:28:20:35 | ControlFlowNode for template | This Template depends on $@. | JinjaSsti.py:16:7:16:13 | ControlFlowNode for request | user-provided value | +| MakoSsti.py:9:27:9:34 | ControlFlowNode for template | MakoSsti.py:6:10:6:16 | ControlFlowNode for request | MakoSsti.py:9:27:9:34 | ControlFlowNode for template | This Template depends on $@. | MakoSsti.py:6:10:6:16 | ControlFlowNode for request | user-provided value | +| TRender.py:7:24:7:31 | ControlFlowNode for template | TRender.py:5:13:5:19 | ControlFlowNode for request | TRender.py:7:24:7:31 | ControlFlowNode for template | This Template depends on $@. | TRender.py:5:13:5:19 | ControlFlowNode for request | user-provided value | diff --git a/python/ql/test/experimental/query-tests/Security/CWE-074/options b/python/ql/test/experimental/query-tests/Security/CWE-074/options deleted file mode 100644 index 2f457593f2e..00000000000 --- a/python/ql/test/experimental/query-tests/Security/CWE-074/options +++ /dev/null @@ -1 +0,0 @@ -semmle-extractor-options: --max-import-depth=3 -p ../../../../query-tests/Security/lib/ From 0336c768714b54e01e1b0f14f53e6d3e4b989daf Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Thu, 17 Aug 2023 15:34:11 +0200 Subject: [PATCH 015/122] Python: Rename template injection tests --- .../{CWE-074 => CWE-074-TemplateInjection}/AirspeedSsti.py | 0 .../Security/{CWE-074 => CWE-074-TemplateInjection}/BottleSsti.py | 0 .../Security/{CWE-074 => CWE-074-TemplateInjection}/Chameleon.py | 0 .../{CWE-074 => CWE-074-TemplateInjection}/ChevronSsti.py | 0 .../{CWE-074 => CWE-074-TemplateInjection}/DjangoTemplates.py | 0 .../{CWE-074 => CWE-074-TemplateInjection}/FlaskTemplate.py | 0 .../Security/{CWE-074 => CWE-074-TemplateInjection}/Genshi.py | 0 .../Security/{CWE-074 => CWE-074-TemplateInjection}/JinjaSsti.py | 0 .../Security/{CWE-074 => CWE-074-TemplateInjection}/MakoSsti.py | 0 .../Security/{CWE-074 => CWE-074-TemplateInjection}/TRender.py | 0 .../TemplateInjection.expected | 0 .../TemplateInjection.qlref | 0 .../paramiko/paramiko.expected | 0 .../{CWE-074 => CWE-074-TemplateInjection}/paramiko/paramiko.py | 0 .../paramiko/paramiko.qlref | 0 15 files changed, 0 insertions(+), 0 deletions(-) rename python/ql/test/experimental/query-tests/Security/{CWE-074 => CWE-074-TemplateInjection}/AirspeedSsti.py (100%) rename python/ql/test/experimental/query-tests/Security/{CWE-074 => CWE-074-TemplateInjection}/BottleSsti.py (100%) rename python/ql/test/experimental/query-tests/Security/{CWE-074 => CWE-074-TemplateInjection}/Chameleon.py (100%) rename python/ql/test/experimental/query-tests/Security/{CWE-074 => CWE-074-TemplateInjection}/ChevronSsti.py (100%) rename python/ql/test/experimental/query-tests/Security/{CWE-074 => CWE-074-TemplateInjection}/DjangoTemplates.py (100%) rename python/ql/test/experimental/query-tests/Security/{CWE-074 => CWE-074-TemplateInjection}/FlaskTemplate.py (100%) rename python/ql/test/experimental/query-tests/Security/{CWE-074 => CWE-074-TemplateInjection}/Genshi.py (100%) rename python/ql/test/experimental/query-tests/Security/{CWE-074 => CWE-074-TemplateInjection}/JinjaSsti.py (100%) rename python/ql/test/experimental/query-tests/Security/{CWE-074 => CWE-074-TemplateInjection}/MakoSsti.py (100%) rename python/ql/test/experimental/query-tests/Security/{CWE-074 => CWE-074-TemplateInjection}/TRender.py (100%) rename python/ql/test/experimental/query-tests/Security/{CWE-074 => CWE-074-TemplateInjection}/TemplateInjection.expected (100%) rename python/ql/test/experimental/query-tests/Security/{CWE-074 => CWE-074-TemplateInjection}/TemplateInjection.qlref (100%) rename python/ql/test/experimental/query-tests/Security/{CWE-074 => CWE-074-TemplateInjection}/paramiko/paramiko.expected (100%) rename python/ql/test/experimental/query-tests/Security/{CWE-074 => CWE-074-TemplateInjection}/paramiko/paramiko.py (100%) rename python/ql/test/experimental/query-tests/Security/{CWE-074 => CWE-074-TemplateInjection}/paramiko/paramiko.qlref (100%) diff --git a/python/ql/test/experimental/query-tests/Security/CWE-074/AirspeedSsti.py b/python/ql/test/experimental/query-tests/Security/CWE-074-TemplateInjection/AirspeedSsti.py similarity index 100% rename from python/ql/test/experimental/query-tests/Security/CWE-074/AirspeedSsti.py rename to python/ql/test/experimental/query-tests/Security/CWE-074-TemplateInjection/AirspeedSsti.py diff --git a/python/ql/test/experimental/query-tests/Security/CWE-074/BottleSsti.py b/python/ql/test/experimental/query-tests/Security/CWE-074-TemplateInjection/BottleSsti.py similarity index 100% rename from python/ql/test/experimental/query-tests/Security/CWE-074/BottleSsti.py rename to python/ql/test/experimental/query-tests/Security/CWE-074-TemplateInjection/BottleSsti.py diff --git a/python/ql/test/experimental/query-tests/Security/CWE-074/Chameleon.py b/python/ql/test/experimental/query-tests/Security/CWE-074-TemplateInjection/Chameleon.py similarity index 100% rename from python/ql/test/experimental/query-tests/Security/CWE-074/Chameleon.py rename to python/ql/test/experimental/query-tests/Security/CWE-074-TemplateInjection/Chameleon.py diff --git a/python/ql/test/experimental/query-tests/Security/CWE-074/ChevronSsti.py b/python/ql/test/experimental/query-tests/Security/CWE-074-TemplateInjection/ChevronSsti.py similarity index 100% rename from python/ql/test/experimental/query-tests/Security/CWE-074/ChevronSsti.py rename to python/ql/test/experimental/query-tests/Security/CWE-074-TemplateInjection/ChevronSsti.py diff --git a/python/ql/test/experimental/query-tests/Security/CWE-074/DjangoTemplates.py b/python/ql/test/experimental/query-tests/Security/CWE-074-TemplateInjection/DjangoTemplates.py similarity index 100% rename from python/ql/test/experimental/query-tests/Security/CWE-074/DjangoTemplates.py rename to python/ql/test/experimental/query-tests/Security/CWE-074-TemplateInjection/DjangoTemplates.py diff --git a/python/ql/test/experimental/query-tests/Security/CWE-074/FlaskTemplate.py b/python/ql/test/experimental/query-tests/Security/CWE-074-TemplateInjection/FlaskTemplate.py similarity index 100% rename from python/ql/test/experimental/query-tests/Security/CWE-074/FlaskTemplate.py rename to python/ql/test/experimental/query-tests/Security/CWE-074-TemplateInjection/FlaskTemplate.py diff --git a/python/ql/test/experimental/query-tests/Security/CWE-074/Genshi.py b/python/ql/test/experimental/query-tests/Security/CWE-074-TemplateInjection/Genshi.py similarity index 100% rename from python/ql/test/experimental/query-tests/Security/CWE-074/Genshi.py rename to python/ql/test/experimental/query-tests/Security/CWE-074-TemplateInjection/Genshi.py diff --git a/python/ql/test/experimental/query-tests/Security/CWE-074/JinjaSsti.py b/python/ql/test/experimental/query-tests/Security/CWE-074-TemplateInjection/JinjaSsti.py similarity index 100% rename from python/ql/test/experimental/query-tests/Security/CWE-074/JinjaSsti.py rename to python/ql/test/experimental/query-tests/Security/CWE-074-TemplateInjection/JinjaSsti.py diff --git a/python/ql/test/experimental/query-tests/Security/CWE-074/MakoSsti.py b/python/ql/test/experimental/query-tests/Security/CWE-074-TemplateInjection/MakoSsti.py similarity index 100% rename from python/ql/test/experimental/query-tests/Security/CWE-074/MakoSsti.py rename to python/ql/test/experimental/query-tests/Security/CWE-074-TemplateInjection/MakoSsti.py diff --git a/python/ql/test/experimental/query-tests/Security/CWE-074/TRender.py b/python/ql/test/experimental/query-tests/Security/CWE-074-TemplateInjection/TRender.py similarity index 100% rename from python/ql/test/experimental/query-tests/Security/CWE-074/TRender.py rename to python/ql/test/experimental/query-tests/Security/CWE-074-TemplateInjection/TRender.py diff --git a/python/ql/test/experimental/query-tests/Security/CWE-074/TemplateInjection.expected b/python/ql/test/experimental/query-tests/Security/CWE-074-TemplateInjection/TemplateInjection.expected similarity index 100% rename from python/ql/test/experimental/query-tests/Security/CWE-074/TemplateInjection.expected rename to python/ql/test/experimental/query-tests/Security/CWE-074-TemplateInjection/TemplateInjection.expected diff --git a/python/ql/test/experimental/query-tests/Security/CWE-074/TemplateInjection.qlref b/python/ql/test/experimental/query-tests/Security/CWE-074-TemplateInjection/TemplateInjection.qlref similarity index 100% rename from python/ql/test/experimental/query-tests/Security/CWE-074/TemplateInjection.qlref rename to python/ql/test/experimental/query-tests/Security/CWE-074-TemplateInjection/TemplateInjection.qlref diff --git a/python/ql/test/experimental/query-tests/Security/CWE-074/paramiko/paramiko.expected b/python/ql/test/experimental/query-tests/Security/CWE-074-TemplateInjection/paramiko/paramiko.expected similarity index 100% rename from python/ql/test/experimental/query-tests/Security/CWE-074/paramiko/paramiko.expected rename to python/ql/test/experimental/query-tests/Security/CWE-074-TemplateInjection/paramiko/paramiko.expected diff --git a/python/ql/test/experimental/query-tests/Security/CWE-074/paramiko/paramiko.py b/python/ql/test/experimental/query-tests/Security/CWE-074-TemplateInjection/paramiko/paramiko.py similarity index 100% rename from python/ql/test/experimental/query-tests/Security/CWE-074/paramiko/paramiko.py rename to python/ql/test/experimental/query-tests/Security/CWE-074-TemplateInjection/paramiko/paramiko.py diff --git a/python/ql/test/experimental/query-tests/Security/CWE-074/paramiko/paramiko.qlref b/python/ql/test/experimental/query-tests/Security/CWE-074-TemplateInjection/paramiko/paramiko.qlref similarity index 100% rename from python/ql/test/experimental/query-tests/Security/CWE-074/paramiko/paramiko.qlref rename to python/ql/test/experimental/query-tests/Security/CWE-074-TemplateInjection/paramiko/paramiko.qlref From 779fe6498c2c5f41eed99503771fb90df6626845 Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Thu, 17 Aug 2023 15:28:26 +0200 Subject: [PATCH 016/122] Python: Rename to `XsltInjection.ql` --- .../Security/CWE-091/{Xslt.qhelp => XsltInjection.qhelp} | 0 .../experimental/Security/CWE-091/{Xslt.ql => XsltInjection.ql} | 0 .../XsltInjection.expected} | 0 .../Security/CWE-091-XsltInjection/XsltInjection.qlref | 1 + .../{CWE-091 => CWE-091-XsltInjection}/XsltSinks.expected | 0 .../Security/{CWE-091 => CWE-091-XsltInjection}/XsltSinks.ql | 0 .../Security/{CWE-091 => CWE-091-XsltInjection}/options | 0 .../Security/{CWE-091 => CWE-091-XsltInjection}/xslt.py | 0 .../Security/{CWE-091 => CWE-091-XsltInjection}/xsltInjection.py | 0 .../Security/{CWE-091 => CWE-091-XsltInjection}/xsltSinks.py | 0 .../ql/test/experimental/query-tests/Security/CWE-091/Xslt.qlref | 1 - 11 files changed, 1 insertion(+), 1 deletion(-) rename python/ql/src/experimental/Security/CWE-091/{Xslt.qhelp => XsltInjection.qhelp} (100%) rename python/ql/src/experimental/Security/CWE-091/{Xslt.ql => XsltInjection.ql} (100%) rename python/ql/test/experimental/query-tests/Security/{CWE-091/Xslt.expected => CWE-091-XsltInjection/XsltInjection.expected} (100%) create mode 100644 python/ql/test/experimental/query-tests/Security/CWE-091-XsltInjection/XsltInjection.qlref rename python/ql/test/experimental/query-tests/Security/{CWE-091 => CWE-091-XsltInjection}/XsltSinks.expected (100%) rename python/ql/test/experimental/query-tests/Security/{CWE-091 => CWE-091-XsltInjection}/XsltSinks.ql (100%) rename python/ql/test/experimental/query-tests/Security/{CWE-091 => CWE-091-XsltInjection}/options (100%) rename python/ql/test/experimental/query-tests/Security/{CWE-091 => CWE-091-XsltInjection}/xslt.py (100%) rename python/ql/test/experimental/query-tests/Security/{CWE-091 => CWE-091-XsltInjection}/xsltInjection.py (100%) rename python/ql/test/experimental/query-tests/Security/{CWE-091 => CWE-091-XsltInjection}/xsltSinks.py (100%) delete mode 100644 python/ql/test/experimental/query-tests/Security/CWE-091/Xslt.qlref diff --git a/python/ql/src/experimental/Security/CWE-091/Xslt.qhelp b/python/ql/src/experimental/Security/CWE-091/XsltInjection.qhelp similarity index 100% rename from python/ql/src/experimental/Security/CWE-091/Xslt.qhelp rename to python/ql/src/experimental/Security/CWE-091/XsltInjection.qhelp diff --git a/python/ql/src/experimental/Security/CWE-091/Xslt.ql b/python/ql/src/experimental/Security/CWE-091/XsltInjection.ql similarity index 100% rename from python/ql/src/experimental/Security/CWE-091/Xslt.ql rename to python/ql/src/experimental/Security/CWE-091/XsltInjection.ql diff --git a/python/ql/test/experimental/query-tests/Security/CWE-091/Xslt.expected b/python/ql/test/experimental/query-tests/Security/CWE-091-XsltInjection/XsltInjection.expected similarity index 100% rename from python/ql/test/experimental/query-tests/Security/CWE-091/Xslt.expected rename to python/ql/test/experimental/query-tests/Security/CWE-091-XsltInjection/XsltInjection.expected diff --git a/python/ql/test/experimental/query-tests/Security/CWE-091-XsltInjection/XsltInjection.qlref b/python/ql/test/experimental/query-tests/Security/CWE-091-XsltInjection/XsltInjection.qlref new file mode 100644 index 00000000000..4d432d461ca --- /dev/null +++ b/python/ql/test/experimental/query-tests/Security/CWE-091-XsltInjection/XsltInjection.qlref @@ -0,0 +1 @@ +experimental/Security/CWE-091/XsltInjection.ql diff --git a/python/ql/test/experimental/query-tests/Security/CWE-091/XsltSinks.expected b/python/ql/test/experimental/query-tests/Security/CWE-091-XsltInjection/XsltSinks.expected similarity index 100% rename from python/ql/test/experimental/query-tests/Security/CWE-091/XsltSinks.expected rename to python/ql/test/experimental/query-tests/Security/CWE-091-XsltInjection/XsltSinks.expected diff --git a/python/ql/test/experimental/query-tests/Security/CWE-091/XsltSinks.ql b/python/ql/test/experimental/query-tests/Security/CWE-091-XsltInjection/XsltSinks.ql similarity index 100% rename from python/ql/test/experimental/query-tests/Security/CWE-091/XsltSinks.ql rename to python/ql/test/experimental/query-tests/Security/CWE-091-XsltInjection/XsltSinks.ql diff --git a/python/ql/test/experimental/query-tests/Security/CWE-091/options b/python/ql/test/experimental/query-tests/Security/CWE-091-XsltInjection/options similarity index 100% rename from python/ql/test/experimental/query-tests/Security/CWE-091/options rename to python/ql/test/experimental/query-tests/Security/CWE-091-XsltInjection/options diff --git a/python/ql/test/experimental/query-tests/Security/CWE-091/xslt.py b/python/ql/test/experimental/query-tests/Security/CWE-091-XsltInjection/xslt.py similarity index 100% rename from python/ql/test/experimental/query-tests/Security/CWE-091/xslt.py rename to python/ql/test/experimental/query-tests/Security/CWE-091-XsltInjection/xslt.py diff --git a/python/ql/test/experimental/query-tests/Security/CWE-091/xsltInjection.py b/python/ql/test/experimental/query-tests/Security/CWE-091-XsltInjection/xsltInjection.py similarity index 100% rename from python/ql/test/experimental/query-tests/Security/CWE-091/xsltInjection.py rename to python/ql/test/experimental/query-tests/Security/CWE-091-XsltInjection/xsltInjection.py diff --git a/python/ql/test/experimental/query-tests/Security/CWE-091/xsltSinks.py b/python/ql/test/experimental/query-tests/Security/CWE-091-XsltInjection/xsltSinks.py similarity index 100% rename from python/ql/test/experimental/query-tests/Security/CWE-091/xsltSinks.py rename to python/ql/test/experimental/query-tests/Security/CWE-091-XsltInjection/xsltSinks.py diff --git a/python/ql/test/experimental/query-tests/Security/CWE-091/Xslt.qlref b/python/ql/test/experimental/query-tests/Security/CWE-091/Xslt.qlref deleted file mode 100644 index 988d13e98a6..00000000000 --- a/python/ql/test/experimental/query-tests/Security/CWE-091/Xslt.qlref +++ /dev/null @@ -1 +0,0 @@ -experimental/Security/CWE-091/Xslt.ql From ef139f2ee9e5e63d53a80651b127ddec31926e3d Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Thu, 17 Aug 2023 15:32:17 +0200 Subject: [PATCH 017/122] Python: Delete `XsltSinks.ql` test --- .../CWE-091-XsltInjection/XsltSinks.expected | 12 ---- .../CWE-091-XsltInjection/XsltSinks.ql | 6 -- .../CWE-091-XsltInjection/xsltSinks.py | 56 ------------------- 3 files changed, 74 deletions(-) delete mode 100644 python/ql/test/experimental/query-tests/Security/CWE-091-XsltInjection/XsltSinks.expected delete mode 100644 python/ql/test/experimental/query-tests/Security/CWE-091-XsltInjection/XsltSinks.ql delete mode 100644 python/ql/test/experimental/query-tests/Security/CWE-091-XsltInjection/xsltSinks.py diff --git a/python/ql/test/experimental/query-tests/Security/CWE-091-XsltInjection/XsltSinks.expected b/python/ql/test/experimental/query-tests/Security/CWE-091-XsltInjection/XsltSinks.expected deleted file mode 100644 index 7150b3046e2..00000000000 --- a/python/ql/test/experimental/query-tests/Security/CWE-091-XsltInjection/XsltSinks.expected +++ /dev/null @@ -1,12 +0,0 @@ -| xslt.py:14:29:14:37 | lxml.etree.parse.xslt | lxml etree xml | -| xsltInjection.py:12:28:12:36 | lxml.etree.XSLT | lxml etree xml | -| xsltInjection.py:21:29:21:37 | lxml.etree.parse.xslt | lxml etree xml | -| xsltInjection.py:31:24:31:32 | lxml.etree.parse.xslt | lxml etree xml | -| xsltInjection.py:40:24:40:32 | lxml.etree.parse.xslt | lxml etree xml | -| xsltInjection.py:50:24:50:32 | lxml.etree.parse.xslt | lxml etree xml | -| xsltInjection.py:60:24:60:32 | lxml.etree.parse.xslt | lxml etree xml | -| xsltInjection.py:69:24:69:32 | lxml.etree.parse.xslt | lxml etree xml | -| xsltInjection.py:79:24:79:32 | lxml.etree.parse.xslt | lxml etree xml | -| xsltSinks.py:17:28:17:36 | lxml.etree.XSLT | lxml etree xml | -| xsltSinks.py:30:29:30:37 | lxml.etree.parse.xslt | lxml etree xml | -| xsltSinks.py:44:24:44:32 | lxml.etree.parse.xslt | lxml etree xml | diff --git a/python/ql/test/experimental/query-tests/Security/CWE-091-XsltInjection/XsltSinks.ql b/python/ql/test/experimental/query-tests/Security/CWE-091-XsltInjection/XsltSinks.ql deleted file mode 100644 index 6ce8fdc4fb5..00000000000 --- a/python/ql/test/experimental/query-tests/Security/CWE-091-XsltInjection/XsltSinks.ql +++ /dev/null @@ -1,6 +0,0 @@ -import python -import experimental.semmle.python.security.injection.XSLT - -from XsltInjection::XsltInjectionSink sink, TaintKind kind -where sink.sinks(kind) -select sink, kind diff --git a/python/ql/test/experimental/query-tests/Security/CWE-091-XsltInjection/xsltSinks.py b/python/ql/test/experimental/query-tests/Security/CWE-091-XsltInjection/xsltSinks.py deleted file mode 100644 index a82fc0c6c5f..00000000000 --- a/python/ql/test/experimental/query-tests/Security/CWE-091-XsltInjection/xsltSinks.py +++ /dev/null @@ -1,56 +0,0 @@ -from lxml import etree -from io import StringIO - -from django.urls import path -from django.http import HttpResponse -from django.template import Template, Context, Engine, engines - - -def a(request): - xslt_root = etree.XML('''\ - - - - - ''') - transform = etree.XSLT(xslt_root) - - -def b(request): - xslt_root = etree.XML('''\ - - - - - ''') - f = StringIO('') - tree = etree.parse(f) - result_tree = tree.xslt(xslt_root) - - -def c(request): - xslt_root = etree.XML('''\ - - - - - ''') - - f = StringIO('') - tree = etree.parse(f) - result = tree.xslt(xslt_root, a="'A'") - - -urlpatterns = [ - path('a', a), - path('b', b), - path('c', c) -] - -if __name__ == "__main__": - a(None) - b(None) - c(None) From 4c693b4fc39e1837479ab86370a56084ed6d0407 Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Thu, 17 Aug 2023 15:33:01 +0200 Subject: [PATCH 018/122] Python: Port `py/xslt-injection` to new data-flow --- .../Security/CWE-091/XsltConcept.qll | 110 +++++++++++++++ .../Security/CWE-091/XsltInjection.ql | 27 +--- .../CWE-091/XsltInjectionCustomizations.qll | 58 ++++++++ .../Security/CWE-091/XsltInjectionQuery.qll | 24 ++++ .../XsltInjection.expected | 133 ++++++++++++------ .../Security/CWE-091-XsltInjection/options | 1 - 6 files changed, 286 insertions(+), 67 deletions(-) create mode 100644 python/ql/src/experimental/Security/CWE-091/XsltConcept.qll create mode 100644 python/ql/src/experimental/Security/CWE-091/XsltInjectionCustomizations.qll create mode 100644 python/ql/src/experimental/Security/CWE-091/XsltInjectionQuery.qll delete mode 100644 python/ql/test/experimental/query-tests/Security/CWE-091-XsltInjection/options diff --git a/python/ql/src/experimental/Security/CWE-091/XsltConcept.qll b/python/ql/src/experimental/Security/CWE-091/XsltConcept.qll new file mode 100644 index 00000000000..e1d5188a5cb --- /dev/null +++ b/python/ql/src/experimental/Security/CWE-091/XsltConcept.qll @@ -0,0 +1,110 @@ +private import python +private import semmle.python.dataflow.new.DataFlow +private import semmle.python.ApiGraphs + +/** + * A data-flow node that constructs a XSLT transformer. + * + * Extend this class to refine existing API models. If you want to model new APIs, + * extend `TemplateConstruction::Range` instead. + */ +class XsltConstruction extends DataFlow::Node instanceof XsltConstruction::Range { + /** Gets the argument that specifies the XSLT transformer. */ + DataFlow::Node getXsltArg() { result = super.getXsltArg() } +} + +/** Provides a class for modeling new system-command execution APIs. */ +module XsltConstruction { + /** + * A data-flow node that constructs a XSLT transformer. + * + * Extend this class to model new APIs. If you want to refine existing API models, + * extend `XsltConstruction` instead. + */ + abstract class Range extends DataFlow::Node { + /** Gets the argument that specifies the XSLT transformer. */ + abstract DataFlow::Node getXsltArg(); + } +} + +/** + * A data-flow node that executes a XSLT transformer. + * + * Extend this class to refine existing API models. If you want to model new APIs, + * extend `TemplateConstruction::Range` instead. + */ +class XsltExecution extends DataFlow::Node instanceof XsltExecution::Range { + /** Gets the argument that specifies the XSLT transformer. */ + DataFlow::Node getXsltArg() { result = super.getXsltArg() } +} + +/** Provides a class for modeling new system-command execution APIs. */ +module XsltExecution { + /** + * A data-flow node that executes a XSLT transformer. + * + * Extend this class to model new APIs. If you want to refine existing API models, + * extend `XsltExecution` instead. + */ + abstract class Range extends DataFlow::Node { + /** Gets the argument that specifies the XSLT transformer. */ + abstract DataFlow::Node getXsltArg(); + } +} + +// ----------------------------------------------------------------------------- +/** + * A call to `lxml.etree.XSLT`. + * + * ```py + * from lxml import etree + * xslt_tree = etree.parse(...) + * doc = etree.parse(...) + * transform = etree.XSLT(xslt_tree) + * result = transform(doc) + * ``` + */ +class LxmlEtreeXsltCall extends XsltConstruction::Range, API::CallNode { + LxmlEtreeXsltCall() { + this = API::moduleImport("lxml").getMember("etree").getMember("XSLT").getACall() + } + + override DataFlow::Node getXsltArg() { result = this.getParameter(0, "xslt_input").asSink() } +} + +/** + * A call to `.xslt` on an lxml ElementTree object. + * + * ```py + * from lxml import etree + * xslt_tree = etree.parse(...) + * doc = etree.parse(...) + * result = doc.xslt(xslt_tree) + * ``` + */ +class XsltAttributeCall extends XsltExecution::Range, API::CallNode { + XsltAttributeCall() { this = elementTreeConstruction(_).getReturn().getMember("xslt").getACall() } + + override DataFlow::Node getXsltArg() { result = this.getParameter(0, "_xslt").asSink() } +} + +// ----------------------------------------------------------------------------- +API::CallNode elementTreeConstruction(DataFlow::Node inputArg) { + // TODO: If we could, would be nice to model this as flow-summaries. But I'm not sure if we actually can :thinking: + // see https://lxml.de/api/lxml.etree-module.html#fromstring + result = API::moduleImport("lxml").getMember("etree").getMember("fromstring").getACall() and + inputArg = result.getParameter(0, "text").asSink() + or + // see https://lxml.de/api/lxml.etree-module.html#fromstringlist + result = API::moduleImport("lxml").getMember("etree").getMember("fromstringlist").getACall() and + inputArg = result.getParameter(0, "strings").asSink() + or + // TODO: technically we should treat parse differently, since it takes a file as argument + // see https://lxml.de/api/lxml.etree-module.html#parse + result = API::moduleImport("lxml").getMember("etree").getMember("parse").getACall() and + inputArg = result.getParameter(0, "source").asSink() + or + // see https://lxml.de/api/lxml.etree-module.html#XML + result = API::moduleImport("lxml").getMember("etree").getMember("XML").getACall() and + inputArg = result.getParameter(0, "text").asSink() +} diff --git a/python/ql/src/experimental/Security/CWE-091/XsltInjection.ql b/python/ql/src/experimental/Security/CWE-091/XsltInjection.ql index 77f405f5f5a..ca6892269e5 100644 --- a/python/ql/src/experimental/Security/CWE-091/XsltInjection.ql +++ b/python/ql/src/experimental/Security/CWE-091/XsltInjection.ql @@ -12,25 +12,10 @@ */ import python -import semmle.python.security.Paths -/* Sources */ -import semmle.python.web.HttpRequest -/* Sinks */ -import experimental.semmle.python.security.injection.XSLT +import XsltInjectionQuery +import XsltInjectionFlow::PathGraph -class XsltInjectionConfiguration extends TaintTracking::Configuration { - XsltInjectionConfiguration() { this = "XSLT injection configuration" } - - deprecated override predicate isSource(TaintTracking::Source source) { - source instanceof HttpRequestTaintSource - } - - deprecated override predicate isSink(TaintTracking::Sink sink) { - sink instanceof XSLTInjection::XSLTInjectionSink - } -} - -from XsltInjectionConfiguration config, TaintedPathSource src, TaintedPathSink sink -where config.hasFlowPath(src, sink) -select sink.getSink(), src, sink, "This XSLT query depends on $@.", src.getSource(), - "a user-provided value" +from XsltInjectionFlow::PathNode source, XsltInjectionFlow::PathNode sink +where XsltInjectionFlow::flowPath(source, sink) +select sink.getNode(), source, sink, "This XSLT query depends on $@.", source.getNode(), + "user-provided value" diff --git a/python/ql/src/experimental/Security/CWE-091/XsltInjectionCustomizations.qll b/python/ql/src/experimental/Security/CWE-091/XsltInjectionCustomizations.qll new file mode 100644 index 00000000000..bda2fe646c9 --- /dev/null +++ b/python/ql/src/experimental/Security/CWE-091/XsltInjectionCustomizations.qll @@ -0,0 +1,58 @@ +/** + * Provides default sources, sinks and sanitizers for detecting + * "XSLT injection" + * vulnerabilities, as well as extension points for adding your own. + */ + +private import python +private import semmle.python.dataflow.new.DataFlow +private import semmle.python.Concepts +private import semmle.python.dataflow.new.RemoteFlowSources +private import semmle.python.dataflow.new.BarrierGuards +private import XsltConcept + +/** + * Provides default sources, sinks and sanitizers for detecting + * "XSLT injection" + * vulnerabilities, as well as extension points for adding your own. + */ +module XsltInjection { + /** + * A data flow source for "XSLT injection" vulnerabilities. + */ + abstract class Source extends DataFlow::Node { } + + /** + * A data flow sink for "XSLT injection" vulnerabilities. + */ + abstract class Sink extends DataFlow::Node { } + + /** + * A sanitizer for "XSLT injection" vulnerabilities. + */ + abstract class Sanitizer extends DataFlow::Node { } + + /** + * A source of remote user input, considered as a flow source. + */ + class RemoteFlowSourceAsSource extends Source, RemoteFlowSource { } + + /** + * An XSLT construction, considered as a flow sink. + */ + class XsltConstructionAsSink extends Sink { + XsltConstructionAsSink() { this = any(XsltConstruction c).getXsltArg() } + } + + /** + * An XSLT execution, considered as a flow sink. + */ + class XsltExecutionAsSink extends Sink { + XsltExecutionAsSink() { this = any(XsltExecution c).getXsltArg() } + } + + /** + * A comparison with a constant string, considered as a sanitizer-guard. + */ + class StringConstCompareAsSanitizerGuard extends Sanitizer, StringConstCompareBarrier { } +} diff --git a/python/ql/src/experimental/Security/CWE-091/XsltInjectionQuery.qll b/python/ql/src/experimental/Security/CWE-091/XsltInjectionQuery.qll new file mode 100644 index 00000000000..4ecae424ed1 --- /dev/null +++ b/python/ql/src/experimental/Security/CWE-091/XsltInjectionQuery.qll @@ -0,0 +1,24 @@ +/** + * Provides a taint-tracking configuration for detecting "XSLT injection" vulnerabilities. + */ + +private import python +import semmle.python.dataflow.new.DataFlow +import semmle.python.dataflow.new.TaintTracking +import XsltInjectionCustomizations::XsltInjection +import XsltConcept + +module XsltInjectionConfig implements DataFlow::ConfigSig { + predicate isSource(DataFlow::Node node) { node instanceof Source } + + predicate isSink(DataFlow::Node node) { node instanceof Sink } + + predicate isAdditionalFlowStep(DataFlow::Node nodeFrom, DataFlow::Node nodeTo) { + // I considered using a FlowState of (raw-string, ElementTree), but in all honesty + // valid code would never have direct flow from a string to a sink anyway... so I + // opted for the more simple approach. + nodeTo = elementTreeConstruction(nodeFrom) + } +} + +module XsltInjectionFlow = TaintTracking::Global; diff --git a/python/ql/test/experimental/query-tests/Security/CWE-091-XsltInjection/XsltInjection.expected b/python/ql/test/experimental/query-tests/Security/CWE-091-XsltInjection/XsltInjection.expected index 89f19160f69..5031bce9f78 100644 --- a/python/ql/test/experimental/query-tests/Security/CWE-091-XsltInjection/XsltInjection.expected +++ b/python/ql/test/experimental/query-tests/Security/CWE-091-XsltInjection/XsltInjection.expected @@ -1,47 +1,90 @@ edges -| xslt.py:10:17:10:28 | dict of etree.XML string | xslt.py:10:17:10:43 | etree.XML string | -| xslt.py:10:17:10:28 | dict of etree.XML string | xslt.py:10:17:10:43 | etree.XML string | -| xslt.py:10:17:10:43 | etree.XML string | xslt.py:11:27:11:35 | etree.XML string | -| xslt.py:10:17:10:43 | etree.XML string | xslt.py:11:27:11:35 | etree.XML string | -| xslt.py:11:17:11:36 | lxml etree xml | xslt.py:14:29:14:37 | lxml etree xml | -| xslt.py:11:17:11:36 | lxml etree xml | xslt.py:14:29:14:37 | lxml etree xml | -| xslt.py:11:27:11:35 | etree.XML string | xslt.py:11:17:11:36 | lxml etree xml | -| xslt.py:11:27:11:35 | etree.XML string | xslt.py:11:17:11:36 | lxml etree xml | -| xsltInjection.py:10:17:10:28 | dict of etree.XML string | xsltInjection.py:10:17:10:43 | etree.XML string | -| xsltInjection.py:10:17:10:28 | dict of etree.XML string | xsltInjection.py:10:17:10:43 | etree.XML string | -| xsltInjection.py:10:17:10:43 | etree.XML string | xsltInjection.py:11:27:11:35 | etree.XML string | -| xsltInjection.py:10:17:10:43 | etree.XML string | xsltInjection.py:11:27:11:35 | etree.XML string | -| xsltInjection.py:11:17:11:36 | lxml etree xml | xsltInjection.py:12:28:12:36 | lxml etree xml | -| xsltInjection.py:11:17:11:36 | lxml etree xml | xsltInjection.py:12:28:12:36 | lxml etree xml | -| xsltInjection.py:11:27:11:35 | etree.XML string | xsltInjection.py:11:17:11:36 | lxml etree xml | -| xsltInjection.py:11:27:11:35 | etree.XML string | xsltInjection.py:11:17:11:36 | lxml etree xml | -| xsltInjection.py:17:17:17:28 | dict of etree.XML string | xsltInjection.py:17:17:17:43 | etree.XML string | -| xsltInjection.py:17:17:17:28 | dict of etree.XML string | xsltInjection.py:17:17:17:43 | etree.XML string | -| xsltInjection.py:17:17:17:43 | etree.XML string | xsltInjection.py:18:27:18:35 | etree.XML string | -| xsltInjection.py:17:17:17:43 | etree.XML string | xsltInjection.py:18:27:18:35 | etree.XML string | -| xsltInjection.py:18:17:18:36 | lxml etree xml | xsltInjection.py:21:29:21:37 | lxml etree xml | -| xsltInjection.py:18:17:18:36 | lxml etree xml | xsltInjection.py:21:29:21:37 | lxml etree xml | -| xsltInjection.py:18:27:18:35 | etree.XML string | xsltInjection.py:18:17:18:36 | lxml etree xml | -| xsltInjection.py:18:27:18:35 | etree.XML string | xsltInjection.py:18:17:18:36 | lxml etree xml | -| xsltInjection.py:26:17:26:28 | dict of etree.XML string | xsltInjection.py:26:17:26:43 | etree.XML string | -| xsltInjection.py:26:17:26:28 | dict of etree.XML string | xsltInjection.py:26:17:26:43 | etree.XML string | -| xsltInjection.py:26:17:26:43 | etree.XML string | xsltInjection.py:27:27:27:35 | etree.XML string | -| xsltInjection.py:26:17:26:43 | etree.XML string | xsltInjection.py:27:27:27:35 | etree.XML string | -| xsltInjection.py:27:17:27:36 | lxml etree xml | xsltInjection.py:31:24:31:32 | lxml etree xml | -| xsltInjection.py:27:17:27:36 | lxml etree xml | xsltInjection.py:31:24:31:32 | lxml etree xml | -| xsltInjection.py:27:27:27:35 | etree.XML string | xsltInjection.py:27:17:27:36 | lxml etree xml | -| xsltInjection.py:27:27:27:35 | etree.XML string | xsltInjection.py:27:17:27:36 | lxml etree xml | -| xsltInjection.py:35:17:35:28 | dict of etree.XML string | xsltInjection.py:35:17:35:43 | etree.XML string | -| xsltInjection.py:35:17:35:28 | dict of etree.XML string | xsltInjection.py:35:17:35:43 | etree.XML string | -| xsltInjection.py:35:17:35:43 | etree.XML string | xsltInjection.py:36:34:36:42 | etree.XML string | -| xsltInjection.py:35:17:35:43 | etree.XML string | xsltInjection.py:36:34:36:42 | etree.XML string | -| xsltInjection.py:36:17:36:43 | lxml etree xml | xsltInjection.py:40:24:40:32 | lxml etree xml | -| xsltInjection.py:36:17:36:43 | lxml etree xml | xsltInjection.py:40:24:40:32 | lxml etree xml | -| xsltInjection.py:36:34:36:42 | etree.XML string | xsltInjection.py:36:17:36:43 | lxml etree xml | -| xsltInjection.py:36:34:36:42 | etree.XML string | xsltInjection.py:36:17:36:43 | lxml etree xml | +| xslt.py:3:26:3:32 | ControlFlowNode for ImportMember | xslt.py:3:26:3:32 | GSSA Variable request | +| xslt.py:3:26:3:32 | GSSA Variable request | xslt.py:10:17:10:23 | ControlFlowNode for request | +| xslt.py:10:17:10:23 | ControlFlowNode for request | xslt.py:10:17:10:28 | ControlFlowNode for Attribute | +| xslt.py:10:17:10:28 | ControlFlowNode for Attribute | xslt.py:10:17:10:43 | ControlFlowNode for Attribute() | +| xslt.py:10:17:10:43 | ControlFlowNode for Attribute() | xslt.py:11:27:11:35 | ControlFlowNode for xsltQuery | +| xslt.py:11:17:11:36 | ControlFlowNode for Attribute() | xslt.py:14:29:14:37 | ControlFlowNode for xslt_root | +| xslt.py:11:27:11:35 | ControlFlowNode for xsltQuery | xslt.py:11:17:11:36 | ControlFlowNode for Attribute() | +| xsltInjection.py:3:26:3:32 | ControlFlowNode for ImportMember | xsltInjection.py:3:26:3:32 | GSSA Variable request | +| xsltInjection.py:3:26:3:32 | GSSA Variable request | xsltInjection.py:10:17:10:23 | ControlFlowNode for request | +| xsltInjection.py:3:26:3:32 | GSSA Variable request | xsltInjection.py:17:17:17:23 | ControlFlowNode for request | +| xsltInjection.py:3:26:3:32 | GSSA Variable request | xsltInjection.py:26:17:26:23 | ControlFlowNode for request | +| xsltInjection.py:3:26:3:32 | GSSA Variable request | xsltInjection.py:35:17:35:23 | ControlFlowNode for request | +| xsltInjection.py:3:26:3:32 | GSSA Variable request | xsltInjection.py:44:17:44:23 | ControlFlowNode for request | +| xsltInjection.py:10:17:10:23 | ControlFlowNode for request | xsltInjection.py:10:17:10:28 | ControlFlowNode for Attribute | +| xsltInjection.py:10:17:10:28 | ControlFlowNode for Attribute | xsltInjection.py:10:17:10:43 | ControlFlowNode for Attribute() | +| xsltInjection.py:10:17:10:43 | ControlFlowNode for Attribute() | xsltInjection.py:11:27:11:35 | ControlFlowNode for xsltQuery | +| xsltInjection.py:11:17:11:36 | ControlFlowNode for Attribute() | xsltInjection.py:12:28:12:36 | ControlFlowNode for xslt_root | +| xsltInjection.py:11:27:11:35 | ControlFlowNode for xsltQuery | xsltInjection.py:11:17:11:36 | ControlFlowNode for Attribute() | +| xsltInjection.py:17:17:17:23 | ControlFlowNode for request | xsltInjection.py:17:17:17:28 | ControlFlowNode for Attribute | +| xsltInjection.py:17:17:17:28 | ControlFlowNode for Attribute | xsltInjection.py:17:17:17:43 | ControlFlowNode for Attribute() | +| xsltInjection.py:17:17:17:43 | ControlFlowNode for Attribute() | xsltInjection.py:18:27:18:35 | ControlFlowNode for xsltQuery | +| xsltInjection.py:18:17:18:36 | ControlFlowNode for Attribute() | xsltInjection.py:21:29:21:37 | ControlFlowNode for xslt_root | +| xsltInjection.py:18:27:18:35 | ControlFlowNode for xsltQuery | xsltInjection.py:18:17:18:36 | ControlFlowNode for Attribute() | +| xsltInjection.py:26:17:26:23 | ControlFlowNode for request | xsltInjection.py:26:17:26:28 | ControlFlowNode for Attribute | +| xsltInjection.py:26:17:26:28 | ControlFlowNode for Attribute | xsltInjection.py:26:17:26:43 | ControlFlowNode for Attribute() | +| xsltInjection.py:26:17:26:43 | ControlFlowNode for Attribute() | xsltInjection.py:27:27:27:35 | ControlFlowNode for xsltQuery | +| xsltInjection.py:27:17:27:36 | ControlFlowNode for Attribute() | xsltInjection.py:31:24:31:32 | ControlFlowNode for xslt_root | +| xsltInjection.py:27:27:27:35 | ControlFlowNode for xsltQuery | xsltInjection.py:27:17:27:36 | ControlFlowNode for Attribute() | +| xsltInjection.py:35:17:35:23 | ControlFlowNode for request | xsltInjection.py:35:17:35:28 | ControlFlowNode for Attribute | +| xsltInjection.py:35:17:35:28 | ControlFlowNode for Attribute | xsltInjection.py:35:17:35:43 | ControlFlowNode for Attribute() | +| xsltInjection.py:35:17:35:43 | ControlFlowNode for Attribute() | xsltInjection.py:36:34:36:42 | ControlFlowNode for xsltQuery | +| xsltInjection.py:36:17:36:43 | ControlFlowNode for Attribute() | xsltInjection.py:40:24:40:32 | ControlFlowNode for xslt_root | +| xsltInjection.py:36:34:36:42 | ControlFlowNode for xsltQuery | xsltInjection.py:36:17:36:43 | ControlFlowNode for Attribute() | +| xsltInjection.py:44:17:44:23 | ControlFlowNode for request | xsltInjection.py:44:17:44:28 | ControlFlowNode for Attribute | +| xsltInjection.py:44:17:44:28 | ControlFlowNode for Attribute | xsltInjection.py:44:17:44:43 | ControlFlowNode for Attribute() | +| xsltInjection.py:44:17:44:43 | ControlFlowNode for Attribute() | xsltInjection.py:45:19:45:44 | ControlFlowNode for List | +| xsltInjection.py:45:19:45:44 | ControlFlowNode for List | xsltInjection.py:46:38:46:48 | ControlFlowNode for xsltStrings | +| xsltInjection.py:46:17:46:49 | ControlFlowNode for Attribute() | xsltInjection.py:50:24:50:32 | ControlFlowNode for xslt_root | +| xsltInjection.py:46:38:46:48 | ControlFlowNode for xsltStrings | xsltInjection.py:46:17:46:49 | ControlFlowNode for Attribute() | +nodes +| xslt.py:3:26:3:32 | ControlFlowNode for ImportMember | semmle.label | ControlFlowNode for ImportMember | +| xslt.py:3:26:3:32 | GSSA Variable request | semmle.label | GSSA Variable request | +| xslt.py:10:17:10:23 | ControlFlowNode for request | semmle.label | ControlFlowNode for request | +| xslt.py:10:17:10:28 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute | +| xslt.py:10:17:10:43 | ControlFlowNode for Attribute() | semmle.label | ControlFlowNode for Attribute() | +| xslt.py:11:17:11:36 | ControlFlowNode for Attribute() | semmle.label | ControlFlowNode for Attribute() | +| xslt.py:11:27:11:35 | ControlFlowNode for xsltQuery | semmle.label | ControlFlowNode for xsltQuery | +| xslt.py:14:29:14:37 | ControlFlowNode for xslt_root | semmle.label | ControlFlowNode for xslt_root | +| xsltInjection.py:3:26:3:32 | ControlFlowNode for ImportMember | semmle.label | ControlFlowNode for ImportMember | +| xsltInjection.py:3:26:3:32 | GSSA Variable request | semmle.label | GSSA Variable request | +| xsltInjection.py:10:17:10:23 | ControlFlowNode for request | semmle.label | ControlFlowNode for request | +| xsltInjection.py:10:17:10:28 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute | +| xsltInjection.py:10:17:10:43 | ControlFlowNode for Attribute() | semmle.label | ControlFlowNode for Attribute() | +| xsltInjection.py:11:17:11:36 | ControlFlowNode for Attribute() | semmle.label | ControlFlowNode for Attribute() | +| xsltInjection.py:11:27:11:35 | ControlFlowNode for xsltQuery | semmle.label | ControlFlowNode for xsltQuery | +| xsltInjection.py:12:28:12:36 | ControlFlowNode for xslt_root | semmle.label | ControlFlowNode for xslt_root | +| xsltInjection.py:17:17:17:23 | ControlFlowNode for request | semmle.label | ControlFlowNode for request | +| xsltInjection.py:17:17:17:28 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute | +| xsltInjection.py:17:17:17:43 | ControlFlowNode for Attribute() | semmle.label | ControlFlowNode for Attribute() | +| xsltInjection.py:18:17:18:36 | ControlFlowNode for Attribute() | semmle.label | ControlFlowNode for Attribute() | +| xsltInjection.py:18:27:18:35 | ControlFlowNode for xsltQuery | semmle.label | ControlFlowNode for xsltQuery | +| xsltInjection.py:21:29:21:37 | ControlFlowNode for xslt_root | semmle.label | ControlFlowNode for xslt_root | +| xsltInjection.py:26:17:26:23 | ControlFlowNode for request | semmle.label | ControlFlowNode for request | +| xsltInjection.py:26:17:26:28 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute | +| xsltInjection.py:26:17:26:43 | ControlFlowNode for Attribute() | semmle.label | ControlFlowNode for Attribute() | +| xsltInjection.py:27:17:27:36 | ControlFlowNode for Attribute() | semmle.label | ControlFlowNode for Attribute() | +| xsltInjection.py:27:27:27:35 | ControlFlowNode for xsltQuery | semmle.label | ControlFlowNode for xsltQuery | +| xsltInjection.py:31:24:31:32 | ControlFlowNode for xslt_root | semmle.label | ControlFlowNode for xslt_root | +| xsltInjection.py:35:17:35:23 | ControlFlowNode for request | semmle.label | ControlFlowNode for request | +| xsltInjection.py:35:17:35:28 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute | +| xsltInjection.py:35:17:35:43 | ControlFlowNode for Attribute() | semmle.label | ControlFlowNode for Attribute() | +| xsltInjection.py:36:17:36:43 | ControlFlowNode for Attribute() | semmle.label | ControlFlowNode for Attribute() | +| xsltInjection.py:36:34:36:42 | ControlFlowNode for xsltQuery | semmle.label | ControlFlowNode for xsltQuery | +| xsltInjection.py:40:24:40:32 | ControlFlowNode for xslt_root | semmle.label | ControlFlowNode for xslt_root | +| xsltInjection.py:44:17:44:23 | ControlFlowNode for request | semmle.label | ControlFlowNode for request | +| xsltInjection.py:44:17:44:28 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute | +| xsltInjection.py:44:17:44:43 | ControlFlowNode for Attribute() | semmle.label | ControlFlowNode for Attribute() | +| xsltInjection.py:45:19:45:44 | ControlFlowNode for List | semmle.label | ControlFlowNode for List | +| xsltInjection.py:46:17:46:49 | ControlFlowNode for Attribute() | semmle.label | ControlFlowNode for Attribute() | +| xsltInjection.py:46:38:46:48 | ControlFlowNode for xsltStrings | semmle.label | ControlFlowNode for xsltStrings | +| xsltInjection.py:50:24:50:32 | ControlFlowNode for xslt_root | semmle.label | ControlFlowNode for xslt_root | +subpaths #select -| xslt.py:14:29:14:37 | xslt_root | xslt.py:10:17:10:28 | dict of etree.XML string | xslt.py:14:29:14:37 | lxml etree xml | This XSLT query depends on $@. | xslt.py:10:17:10:28 | Attribute | a user-provided value | -| xsltInjection.py:12:28:12:36 | xslt_root | xsltInjection.py:10:17:10:28 | dict of etree.XML string | xsltInjection.py:12:28:12:36 | lxml etree xml | This XSLT query depends on $@. | xsltInjection.py:10:17:10:28 | Attribute | a user-provided value | -| xsltInjection.py:21:29:21:37 | xslt_root | xsltInjection.py:17:17:17:28 | dict of etree.XML string | xsltInjection.py:21:29:21:37 | lxml etree xml | This XSLT query depends on $@. | xsltInjection.py:17:17:17:28 | Attribute | a user-provided value | -| xsltInjection.py:31:24:31:32 | xslt_root | xsltInjection.py:26:17:26:28 | dict of etree.XML string | xsltInjection.py:31:24:31:32 | lxml etree xml | This XSLT query depends on $@. | xsltInjection.py:26:17:26:28 | Attribute | a user-provided value | -| xsltInjection.py:40:24:40:32 | xslt_root | xsltInjection.py:35:17:35:28 | dict of etree.XML string | xsltInjection.py:40:24:40:32 | lxml etree xml | This XSLT query depends on $@. | xsltInjection.py:35:17:35:28 | Attribute | a user-provided value | +| xslt.py:14:29:14:37 | ControlFlowNode for xslt_root | xslt.py:3:26:3:32 | ControlFlowNode for ImportMember | xslt.py:14:29:14:37 | ControlFlowNode for xslt_root | This XSLT query depends on $@. | xslt.py:3:26:3:32 | ControlFlowNode for ImportMember | user-provided value | +| xsltInjection.py:12:28:12:36 | ControlFlowNode for xslt_root | xsltInjection.py:3:26:3:32 | ControlFlowNode for ImportMember | xsltInjection.py:12:28:12:36 | ControlFlowNode for xslt_root | This XSLT query depends on $@. | xsltInjection.py:3:26:3:32 | ControlFlowNode for ImportMember | user-provided value | +| xsltInjection.py:21:29:21:37 | ControlFlowNode for xslt_root | xsltInjection.py:3:26:3:32 | ControlFlowNode for ImportMember | xsltInjection.py:21:29:21:37 | ControlFlowNode for xslt_root | This XSLT query depends on $@. | xsltInjection.py:3:26:3:32 | ControlFlowNode for ImportMember | user-provided value | +| xsltInjection.py:31:24:31:32 | ControlFlowNode for xslt_root | xsltInjection.py:3:26:3:32 | ControlFlowNode for ImportMember | xsltInjection.py:31:24:31:32 | ControlFlowNode for xslt_root | This XSLT query depends on $@. | xsltInjection.py:3:26:3:32 | ControlFlowNode for ImportMember | user-provided value | +| xsltInjection.py:40:24:40:32 | ControlFlowNode for xslt_root | xsltInjection.py:3:26:3:32 | ControlFlowNode for ImportMember | xsltInjection.py:40:24:40:32 | ControlFlowNode for xslt_root | This XSLT query depends on $@. | xsltInjection.py:3:26:3:32 | ControlFlowNode for ImportMember | user-provided value | +| xsltInjection.py:50:24:50:32 | ControlFlowNode for xslt_root | xsltInjection.py:3:26:3:32 | ControlFlowNode for ImportMember | xsltInjection.py:50:24:50:32 | ControlFlowNode for xslt_root | This XSLT query depends on $@. | xsltInjection.py:3:26:3:32 | ControlFlowNode for ImportMember | user-provided value | diff --git a/python/ql/test/experimental/query-tests/Security/CWE-091-XsltInjection/options b/python/ql/test/experimental/query-tests/Security/CWE-091-XsltInjection/options deleted file mode 100644 index 2f457593f2e..00000000000 --- a/python/ql/test/experimental/query-tests/Security/CWE-091-XsltInjection/options +++ /dev/null @@ -1 +0,0 @@ -semmle-extractor-options: --max-import-depth=3 -p ../../../../query-tests/Security/lib/ From cf54d3f4caf1fdede4a58e55f974c9876728e553 Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Thu, 17 Aug 2023 15:45:28 +0200 Subject: [PATCH 019/122] Python: Move paramiko tests to own folder --- .../paramiko => CWE-074-paramiko}/paramiko.expected | 0 .../paramiko => CWE-074-paramiko}/paramiko.py | 0 .../paramiko => CWE-074-paramiko}/paramiko.qlref | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename python/ql/test/experimental/query-tests/Security/{CWE-074-TemplateInjection/paramiko => CWE-074-paramiko}/paramiko.expected (100%) rename python/ql/test/experimental/query-tests/Security/{CWE-074-TemplateInjection/paramiko => CWE-074-paramiko}/paramiko.py (100%) rename python/ql/test/experimental/query-tests/Security/{CWE-074-TemplateInjection/paramiko => CWE-074-paramiko}/paramiko.qlref (100%) diff --git a/python/ql/test/experimental/query-tests/Security/CWE-074-TemplateInjection/paramiko/paramiko.expected b/python/ql/test/experimental/query-tests/Security/CWE-074-paramiko/paramiko.expected similarity index 100% rename from python/ql/test/experimental/query-tests/Security/CWE-074-TemplateInjection/paramiko/paramiko.expected rename to python/ql/test/experimental/query-tests/Security/CWE-074-paramiko/paramiko.expected diff --git a/python/ql/test/experimental/query-tests/Security/CWE-074-TemplateInjection/paramiko/paramiko.py b/python/ql/test/experimental/query-tests/Security/CWE-074-paramiko/paramiko.py similarity index 100% rename from python/ql/test/experimental/query-tests/Security/CWE-074-TemplateInjection/paramiko/paramiko.py rename to python/ql/test/experimental/query-tests/Security/CWE-074-paramiko/paramiko.py diff --git a/python/ql/test/experimental/query-tests/Security/CWE-074-TemplateInjection/paramiko/paramiko.qlref b/python/ql/test/experimental/query-tests/Security/CWE-074-paramiko/paramiko.qlref similarity index 100% rename from python/ql/test/experimental/query-tests/Security/CWE-074-TemplateInjection/paramiko/paramiko.qlref rename to python/ql/test/experimental/query-tests/Security/CWE-074-paramiko/paramiko.qlref From 33f8998c2eb8884d323e48d188a08ac42554b7ce Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Fri, 18 Aug 2023 10:19:44 +0200 Subject: [PATCH 020/122] Python: Minor fix in test --- .../Security/CWE-074-TemplateInjection/ChevronSsti.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/ql/test/experimental/query-tests/Security/CWE-074-TemplateInjection/ChevronSsti.py b/python/ql/test/experimental/query-tests/Security/CWE-074-TemplateInjection/ChevronSsti.py index f2430e437ae..f3b0e57fc8f 100644 --- a/python/ql/test/experimental/query-tests/Security/CWE-074-TemplateInjection/ChevronSsti.py +++ b/python/ql/test/experimental/query-tests/Security/CWE-074-TemplateInjection/ChevronSsti.py @@ -5,13 +5,13 @@ import chevron app = Flask(__name__) -@route('/other') +@app.route('/other') def a(): template = request.args.get('template') return chevron.render(template, {"key": "value"}) -@route('/other2') +@app.route('/other2') def b(): template = request.args.get('template') args = { From 38577e6a5c551feedc50dc9e2b3a00dd3a7cf403 Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Fri, 18 Aug 2023 10:20:16 +0200 Subject: [PATCH 021/122] Python: Remove duplicated SSTI tests Besides the Cheetah tests, which were missing from the query tests. --- .../CWE-074-TemplateInjection/CheetahSinks.py | 22 +++++++++++ .../semmle/python/templates/Airspeed.py | 10 ----- .../templates/AirspeedSSTISinks.expected | 2 - .../python/templates/AirspeedSSTISinks.ql | 5 --- .../semmle/python/templates/Bottle.py | 17 -------- .../python/templates/BottleSSTISinks.expected | 3 -- .../python/templates/BottleSSTISinks.ql | 5 --- .../semmle/python/templates/Chameleon.py | 5 --- .../templates/ChameleonSSTISinks.expected | 2 - .../python/templates/ChameleonSSTISinks.ql | 5 --- .../templates/CheetahSSTISinks.expected | 3 -- .../python/templates/CheetahSSTISinks.ql | 5 --- .../semmle/python/templates/CheetahSinks.py | 20 ---------- .../templates/ChevronSSTISinks.expected | 2 - .../python/templates/ChevronSSTISinks.ql | 5 --- .../semmle/python/templates/ChevronSinks.py | 22 ----------- .../python/templates/DjangoSSTISinks.expected | 2 - .../python/templates/DjangoSSTISinks.ql | 5 --- .../python/templates/DjangoTemplates.py | 39 ------------------- .../semmle/python/templates/Genshi.py | 10 ----- .../python/templates/GenshiSSTISinks.expected | 3 -- .../python/templates/GenshiSSTISinks.ql | 5 --- .../python/templates/Jinja2Templates.py | 17 -------- .../python/templates/JinjaSSTISinks.expected | 4 -- .../semmle/python/templates/JinjaSSTISinks.ql | 5 --- .../semmle/python/templates/Mako.py | 5 --- .../python/templates/MakoSSTISinks.expected | 2 - .../semmle/python/templates/MakoSSTISinks.ql | 5 --- .../semmle/python/templates/TRender.py | 6 --- .../templates/TRenderSSTISinks.expected | 2 - .../python/templates/TRenderSSTISinks.ql | 5 --- .../semmle/python/templates/options | 1 - 32 files changed, 22 insertions(+), 227 deletions(-) create mode 100644 python/ql/test/experimental/query-tests/Security/CWE-074-TemplateInjection/CheetahSinks.py delete mode 100644 python/ql/test/experimental/semmle/python/templates/Airspeed.py delete mode 100644 python/ql/test/experimental/semmle/python/templates/AirspeedSSTISinks.expected delete mode 100644 python/ql/test/experimental/semmle/python/templates/AirspeedSSTISinks.ql delete mode 100644 python/ql/test/experimental/semmle/python/templates/Bottle.py delete mode 100644 python/ql/test/experimental/semmle/python/templates/BottleSSTISinks.expected delete mode 100644 python/ql/test/experimental/semmle/python/templates/BottleSSTISinks.ql delete mode 100644 python/ql/test/experimental/semmle/python/templates/Chameleon.py delete mode 100644 python/ql/test/experimental/semmle/python/templates/ChameleonSSTISinks.expected delete mode 100644 python/ql/test/experimental/semmle/python/templates/ChameleonSSTISinks.ql delete mode 100644 python/ql/test/experimental/semmle/python/templates/CheetahSSTISinks.expected delete mode 100644 python/ql/test/experimental/semmle/python/templates/CheetahSSTISinks.ql delete mode 100644 python/ql/test/experimental/semmle/python/templates/CheetahSinks.py delete mode 100644 python/ql/test/experimental/semmle/python/templates/ChevronSSTISinks.expected delete mode 100644 python/ql/test/experimental/semmle/python/templates/ChevronSSTISinks.ql delete mode 100644 python/ql/test/experimental/semmle/python/templates/ChevronSinks.py delete mode 100644 python/ql/test/experimental/semmle/python/templates/DjangoSSTISinks.expected delete mode 100644 python/ql/test/experimental/semmle/python/templates/DjangoSSTISinks.ql delete mode 100644 python/ql/test/experimental/semmle/python/templates/DjangoTemplates.py delete mode 100644 python/ql/test/experimental/semmle/python/templates/Genshi.py delete mode 100644 python/ql/test/experimental/semmle/python/templates/GenshiSSTISinks.expected delete mode 100644 python/ql/test/experimental/semmle/python/templates/GenshiSSTISinks.ql delete mode 100644 python/ql/test/experimental/semmle/python/templates/Jinja2Templates.py delete mode 100644 python/ql/test/experimental/semmle/python/templates/JinjaSSTISinks.expected delete mode 100644 python/ql/test/experimental/semmle/python/templates/JinjaSSTISinks.ql delete mode 100644 python/ql/test/experimental/semmle/python/templates/Mako.py delete mode 100644 python/ql/test/experimental/semmle/python/templates/MakoSSTISinks.expected delete mode 100644 python/ql/test/experimental/semmle/python/templates/MakoSSTISinks.ql delete mode 100644 python/ql/test/experimental/semmle/python/templates/TRender.py delete mode 100644 python/ql/test/experimental/semmle/python/templates/TRenderSSTISinks.expected delete mode 100644 python/ql/test/experimental/semmle/python/templates/TRenderSSTISinks.ql delete mode 100644 python/ql/test/experimental/semmle/python/templates/options diff --git a/python/ql/test/experimental/query-tests/Security/CWE-074-TemplateInjection/CheetahSinks.py b/python/ql/test/experimental/query-tests/Security/CWE-074-TemplateInjection/CheetahSinks.py new file mode 100644 index 00000000000..7f9fed4decf --- /dev/null +++ b/python/ql/test/experimental/query-tests/Security/CWE-074-TemplateInjection/CheetahSinks.py @@ -0,0 +1,22 @@ +from flask import Flask, request +from Cheetah.Template import Template + + +app = Flask(__name__) + + +@app.route('/other') +def a(): + template = request.args.get('template') + return Template(template) + + +class Template3(Template): + title = 'Hello World Example!' + contents = 'Hello World!' + + +@app.route('/other2') +def b(): + template = request.args.get('template') + t3 = Template3(template) diff --git a/python/ql/test/experimental/semmle/python/templates/Airspeed.py b/python/ql/test/experimental/semmle/python/templates/Airspeed.py deleted file mode 100644 index a41d70432a0..00000000000 --- a/python/ql/test/experimental/semmle/python/templates/Airspeed.py +++ /dev/null @@ -1,10 +0,0 @@ -from bottle import Bottle, route, request, redirect, response -import airspeed - - -app = Bottle() - - -@route('/other') -def a(): - return airspeed.Template("sink") diff --git a/python/ql/test/experimental/semmle/python/templates/AirspeedSSTISinks.expected b/python/ql/test/experimental/semmle/python/templates/AirspeedSSTISinks.expected deleted file mode 100644 index e938211434c..00000000000 --- a/python/ql/test/experimental/semmle/python/templates/AirspeedSSTISinks.expected +++ /dev/null @@ -1,2 +0,0 @@ -WARNING: Type SSTISink has been deprecated and may be removed in future (AirspeedSSTISinks.ql:4,6-14) -| Airspeed.py:10:30:10:35 | argument to airspeed.Template() | diff --git a/python/ql/test/experimental/semmle/python/templates/AirspeedSSTISinks.ql b/python/ql/test/experimental/semmle/python/templates/AirspeedSSTISinks.ql deleted file mode 100644 index e9c51ef11ad..00000000000 --- a/python/ql/test/experimental/semmle/python/templates/AirspeedSSTISinks.ql +++ /dev/null @@ -1,5 +0,0 @@ -import python -import experimental.semmle.python.templates.Airspeed - -from SSTISink s -select s diff --git a/python/ql/test/experimental/semmle/python/templates/Bottle.py b/python/ql/test/experimental/semmle/python/templates/Bottle.py deleted file mode 100644 index f6b2fec090e..00000000000 --- a/python/ql/test/experimental/semmle/python/templates/Bottle.py +++ /dev/null @@ -1,17 +0,0 @@ -from bottle import Bottle, route, request, redirect, response, SimpleTemplate -from bottle import template as temp - - -app = Bottle() - - -@route('/other') -def a(): - template = "test" - tpl = SimpleTemplate(template) - - -@route('/other2') -def b(): - template = "test" - return temp(template, name='World') diff --git a/python/ql/test/experimental/semmle/python/templates/BottleSSTISinks.expected b/python/ql/test/experimental/semmle/python/templates/BottleSSTISinks.expected deleted file mode 100644 index 1802708c2de..00000000000 --- a/python/ql/test/experimental/semmle/python/templates/BottleSSTISinks.expected +++ /dev/null @@ -1,3 +0,0 @@ -WARNING: Type SSTISink has been deprecated and may be removed in future (BottleSSTISinks.ql:4,6-14) -| Bottle.py:11:26:11:33 | argument to bottle.SimpleTemplate() | -| Bottle.py:17:17:17:24 | argument to bottle.template() | diff --git a/python/ql/test/experimental/semmle/python/templates/BottleSSTISinks.ql b/python/ql/test/experimental/semmle/python/templates/BottleSSTISinks.ql deleted file mode 100644 index c0ba59ef957..00000000000 --- a/python/ql/test/experimental/semmle/python/templates/BottleSSTISinks.ql +++ /dev/null @@ -1,5 +0,0 @@ -import python -import experimental.semmle.python.templates.Bottle - -from SSTISink s -select s diff --git a/python/ql/test/experimental/semmle/python/templates/Chameleon.py b/python/ql/test/experimental/semmle/python/templates/Chameleon.py deleted file mode 100644 index 6d96f0752a9..00000000000 --- a/python/ql/test/experimental/semmle/python/templates/Chameleon.py +++ /dev/null @@ -1,5 +0,0 @@ -from chameleon import PageTemplate - - -def chameleon(): - template = PageTemplate("sink") diff --git a/python/ql/test/experimental/semmle/python/templates/ChameleonSSTISinks.expected b/python/ql/test/experimental/semmle/python/templates/ChameleonSSTISinks.expected deleted file mode 100644 index d6a46986f11..00000000000 --- a/python/ql/test/experimental/semmle/python/templates/ChameleonSSTISinks.expected +++ /dev/null @@ -1,2 +0,0 @@ -WARNING: Type SSTISink has been deprecated and may be removed in future (ChameleonSSTISinks.ql:4,6-14) -| Chameleon.py:5:29:5:34 | argument to Chameleon.PageTemplate() | diff --git a/python/ql/test/experimental/semmle/python/templates/ChameleonSSTISinks.ql b/python/ql/test/experimental/semmle/python/templates/ChameleonSSTISinks.ql deleted file mode 100644 index ee9d41434af..00000000000 --- a/python/ql/test/experimental/semmle/python/templates/ChameleonSSTISinks.ql +++ /dev/null @@ -1,5 +0,0 @@ -import python -import experimental.semmle.python.templates.Chameleon - -from SSTISink s -select s diff --git a/python/ql/test/experimental/semmle/python/templates/CheetahSSTISinks.expected b/python/ql/test/experimental/semmle/python/templates/CheetahSSTISinks.expected deleted file mode 100644 index 3971b25e356..00000000000 --- a/python/ql/test/experimental/semmle/python/templates/CheetahSSTISinks.expected +++ /dev/null @@ -1,3 +0,0 @@ -WARNING: Type SSTISink has been deprecated and may be removed in future (CheetahSSTISinks.ql:4,6-14) -| CheetahSinks.py:10:21:10:26 | argument to Cheetah.Template.Template() | -| CheetahSinks.py:20:20:20:25 | argument to Cheetah.Template.Template() | diff --git a/python/ql/test/experimental/semmle/python/templates/CheetahSSTISinks.ql b/python/ql/test/experimental/semmle/python/templates/CheetahSSTISinks.ql deleted file mode 100644 index 10c6c79a4d5..00000000000 --- a/python/ql/test/experimental/semmle/python/templates/CheetahSSTISinks.ql +++ /dev/null @@ -1,5 +0,0 @@ -import python -import experimental.semmle.python.templates.Cheetah - -from SSTISink s -select s diff --git a/python/ql/test/experimental/semmle/python/templates/CheetahSinks.py b/python/ql/test/experimental/semmle/python/templates/CheetahSinks.py deleted file mode 100644 index 0bb3364a178..00000000000 --- a/python/ql/test/experimental/semmle/python/templates/CheetahSinks.py +++ /dev/null @@ -1,20 +0,0 @@ -from bottle import Bottle, route, request, redirect, response, SimpleTemplate -from Cheetah.Template import Template - - -app = Bottle() - - -@route('/other') -def a(): - return Template("sink") - - -class Template3(Template): - title = 'Hello World Example!' - contents = 'Hello World!' - - -@route('/other2') -def b(): - t3 = Template3("sink") diff --git a/python/ql/test/experimental/semmle/python/templates/ChevronSSTISinks.expected b/python/ql/test/experimental/semmle/python/templates/ChevronSSTISinks.expected deleted file mode 100644 index 50ebb008209..00000000000 --- a/python/ql/test/experimental/semmle/python/templates/ChevronSSTISinks.expected +++ /dev/null @@ -1,2 +0,0 @@ -WARNING: Type SSTISink has been deprecated and may be removed in future (ChevronSSTISinks.ql:4,6-14) -| ChevronSinks.py:10:27:10:32 | argument to chevron.render() | diff --git a/python/ql/test/experimental/semmle/python/templates/ChevronSSTISinks.ql b/python/ql/test/experimental/semmle/python/templates/ChevronSSTISinks.ql deleted file mode 100644 index 545c1f8f79a..00000000000 --- a/python/ql/test/experimental/semmle/python/templates/ChevronSSTISinks.ql +++ /dev/null @@ -1,5 +0,0 @@ -import python -import experimental.semmle.python.templates.Chevron - -from SSTISink s -select s diff --git a/python/ql/test/experimental/semmle/python/templates/ChevronSinks.py b/python/ql/test/experimental/semmle/python/templates/ChevronSinks.py deleted file mode 100644 index 26d35708bb6..00000000000 --- a/python/ql/test/experimental/semmle/python/templates/ChevronSinks.py +++ /dev/null @@ -1,22 +0,0 @@ -from bottle import Bottle, route, request, redirect, response, SimpleTemplate -import chevron - - -app = Bottle() - - -@route('/other') -def a(): - return chevron.render("sink", {"key": "value"}) - - -@route('/other2') -def b(): - sink = { - 'template': "template", - - 'data': { - 'key': 'value' - } - } - return chevron.render(**sink) diff --git a/python/ql/test/experimental/semmle/python/templates/DjangoSSTISinks.expected b/python/ql/test/experimental/semmle/python/templates/DjangoSSTISinks.expected deleted file mode 100644 index a38fdbc323f..00000000000 --- a/python/ql/test/experimental/semmle/python/templates/DjangoSSTISinks.expected +++ /dev/null @@ -1,2 +0,0 @@ -WARNING: Type SSTISink has been deprecated and may be removed in future (DjangoSSTISinks.ql:4,6-14) -| DjangoTemplates.py:9:18:9:25 | argument to Django.template() | diff --git a/python/ql/test/experimental/semmle/python/templates/DjangoSSTISinks.ql b/python/ql/test/experimental/semmle/python/templates/DjangoSSTISinks.ql deleted file mode 100644 index eecd31aeb87..00000000000 --- a/python/ql/test/experimental/semmle/python/templates/DjangoSSTISinks.ql +++ /dev/null @@ -1,5 +0,0 @@ -import python -import experimental.semmle.python.templates.DjangoTemplate - -from SSTISink s -select s diff --git a/python/ql/test/experimental/semmle/python/templates/DjangoTemplates.py b/python/ql/test/experimental/semmle/python/templates/DjangoTemplates.py deleted file mode 100644 index 981109bf7dc..00000000000 --- a/python/ql/test/experimental/semmle/python/templates/DjangoTemplates.py +++ /dev/null @@ -1,39 +0,0 @@ -from django.urls import path -from django.http import HttpResponse -from django.template import Template, Context, Engine, engines - - -def dj(request): - # Load the template - template = request.GET['template'] - t = Template(template) - ctx = Context(locals()) - html = t.render(ctx) - return HttpResponse(html) - - -def djEngine(request): - # Load the template - template = request.GET['template'] - - django_engine = engines['django'] - t = django_engine.from_string(template) - ctx = Context(locals()) - html = t.render(ctx) - return HttpResponse(html) - - -def djEngineJinja(request): - # Load the template - template = request.GET['template'] - - django_engine = engines['jinja'] - t = django_engine.from_string(template) - ctx = Context(locals()) - html = t.render(ctx) - return HttpResponse(html) - - -urlpatterns = [ - path('', dj) -] diff --git a/python/ql/test/experimental/semmle/python/templates/Genshi.py b/python/ql/test/experimental/semmle/python/templates/Genshi.py deleted file mode 100644 index 7c46a2b31dc..00000000000 --- a/python/ql/test/experimental/semmle/python/templates/Genshi.py +++ /dev/null @@ -1,10 +0,0 @@ - - -def genshi1(): - from genshi.template import MarkupTemplate - tmpl = MarkupTemplate('sink') - - -def genshi2(): - from genshi.template import TextTemplate - tmpl = TextTemplate('sink') \ No newline at end of file diff --git a/python/ql/test/experimental/semmle/python/templates/GenshiSSTISinks.expected b/python/ql/test/experimental/semmle/python/templates/GenshiSSTISinks.expected deleted file mode 100644 index cfc22364413..00000000000 --- a/python/ql/test/experimental/semmle/python/templates/GenshiSSTISinks.expected +++ /dev/null @@ -1,3 +0,0 @@ -WARNING: Type SSTISink has been deprecated and may be removed in future (GenshiSSTISinks.ql:4,6-14) -| Genshi.py:5:27:5:32 | argument to genshi.template.MarkupTemplate() | -| Genshi.py:10:25:10:30 | argument to genshi.template.TextTemplate() | diff --git a/python/ql/test/experimental/semmle/python/templates/GenshiSSTISinks.ql b/python/ql/test/experimental/semmle/python/templates/GenshiSSTISinks.ql deleted file mode 100644 index f0d87e97ec1..00000000000 --- a/python/ql/test/experimental/semmle/python/templates/GenshiSSTISinks.ql +++ /dev/null @@ -1,5 +0,0 @@ -import python -import experimental.semmle.python.templates.Genshi - -from SSTISink s -select s diff --git a/python/ql/test/experimental/semmle/python/templates/Jinja2Templates.py b/python/ql/test/experimental/semmle/python/templates/Jinja2Templates.py deleted file mode 100644 index e52538d4946..00000000000 --- a/python/ql/test/experimental/semmle/python/templates/Jinja2Templates.py +++ /dev/null @@ -1,17 +0,0 @@ -from jinja2 import Template as Jinja2_Template -from jinja2 import Environment, DictLoader, escape - - -def jinja(): - t = Jinja2_Template("sink") - - -def jinja2(): - random = "esdad" + "asdad" - t = Jinja2_Template(random) - - -def jinja3(): - random = 1234 - t = Jinja2_Template("sink"+random) - diff --git a/python/ql/test/experimental/semmle/python/templates/JinjaSSTISinks.expected b/python/ql/test/experimental/semmle/python/templates/JinjaSSTISinks.expected deleted file mode 100644 index 7b91c934947..00000000000 --- a/python/ql/test/experimental/semmle/python/templates/JinjaSSTISinks.expected +++ /dev/null @@ -1,4 +0,0 @@ -WARNING: Type SSTISink has been deprecated and may be removed in future (JinjaSSTISinks.ql:4,6-14) -| Jinja2Templates.py:6:25:6:30 | argument to jinja2.Template() | -| Jinja2Templates.py:11:25:11:30 | argument to jinja2.Template() | -| Jinja2Templates.py:16:25:16:37 | argument to jinja2.Template() | diff --git a/python/ql/test/experimental/semmle/python/templates/JinjaSSTISinks.ql b/python/ql/test/experimental/semmle/python/templates/JinjaSSTISinks.ql deleted file mode 100644 index ca80d8bc570..00000000000 --- a/python/ql/test/experimental/semmle/python/templates/JinjaSSTISinks.ql +++ /dev/null @@ -1,5 +0,0 @@ -import python -import experimental.semmle.python.templates.Jinja - -from SSTISink s -select s diff --git a/python/ql/test/experimental/semmle/python/templates/Mako.py b/python/ql/test/experimental/semmle/python/templates/Mako.py deleted file mode 100644 index 3af60b164ea..00000000000 --- a/python/ql/test/experimental/semmle/python/templates/Mako.py +++ /dev/null @@ -1,5 +0,0 @@ - - -def mako(): - from mako.template import Template - mytemplate = Template("sink") diff --git a/python/ql/test/experimental/semmle/python/templates/MakoSSTISinks.expected b/python/ql/test/experimental/semmle/python/templates/MakoSSTISinks.expected deleted file mode 100644 index 005e14f218a..00000000000 --- a/python/ql/test/experimental/semmle/python/templates/MakoSSTISinks.expected +++ /dev/null @@ -1,2 +0,0 @@ -WARNING: Type SSTISink has been deprecated and may be removed in future (MakoSSTISinks.ql:4,6-14) -| Mako.py:5:27:5:32 | argument to mako.template.Template() | diff --git a/python/ql/test/experimental/semmle/python/templates/MakoSSTISinks.ql b/python/ql/test/experimental/semmle/python/templates/MakoSSTISinks.ql deleted file mode 100644 index eed89420c54..00000000000 --- a/python/ql/test/experimental/semmle/python/templates/MakoSSTISinks.ql +++ /dev/null @@ -1,5 +0,0 @@ -import python -import experimental.semmle.python.templates.Mako - -from SSTISink s -select s diff --git a/python/ql/test/experimental/semmle/python/templates/TRender.py b/python/ql/test/experimental/semmle/python/templates/TRender.py deleted file mode 100644 index 6ed5a799942..00000000000 --- a/python/ql/test/experimental/semmle/python/templates/TRender.py +++ /dev/null @@ -1,6 +0,0 @@ - - -def trender(): - from trender import TRender - template = '@greet world!' - compiled = TRender(template) diff --git a/python/ql/test/experimental/semmle/python/templates/TRenderSSTISinks.expected b/python/ql/test/experimental/semmle/python/templates/TRenderSSTISinks.expected deleted file mode 100644 index 26dea55a6c8..00000000000 --- a/python/ql/test/experimental/semmle/python/templates/TRenderSSTISinks.expected +++ /dev/null @@ -1,2 +0,0 @@ -WARNING: Type SSTISink has been deprecated and may be removed in future (TRenderSSTISinks.ql:4,6-14) -| TRender.py:6:24:6:31 | argument to trender.TRender() | diff --git a/python/ql/test/experimental/semmle/python/templates/TRenderSSTISinks.ql b/python/ql/test/experimental/semmle/python/templates/TRenderSSTISinks.ql deleted file mode 100644 index ec3a1bba57f..00000000000 --- a/python/ql/test/experimental/semmle/python/templates/TRenderSSTISinks.ql +++ /dev/null @@ -1,5 +0,0 @@ -import python -import experimental.semmle.python.templates.TRender - -from SSTISink s -select s diff --git a/python/ql/test/experimental/semmle/python/templates/options b/python/ql/test/experimental/semmle/python/templates/options deleted file mode 100644 index c3bc9413072..00000000000 --- a/python/ql/test/experimental/semmle/python/templates/options +++ /dev/null @@ -1 +0,0 @@ -semmle-extractor-options: --lang=3 --max-import-depth=3 -p ../../../../../query-tests/Security/lib/ From 480e3bf506f1061fee51f3f91b51e5893741cf90 Mon Sep 17 00:00:00 2001 From: Stephan Brandauer Date: Fri, 18 Aug 2023 10:27:46 +0200 Subject: [PATCH 022/122] Java: update model exclusions logic to cope with new automodel test location --- .../lib/semmle/code/java/dataflow/internal/ModelExclusions.qll | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/ql/lib/semmle/code/java/dataflow/internal/ModelExclusions.qll b/java/ql/lib/semmle/code/java/dataflow/internal/ModelExclusions.qll index 745cf7934f0..5f1996989ad 100644 --- a/java/ql/lib/semmle/code/java/dataflow/internal/ModelExclusions.qll +++ b/java/ql/lib/semmle/code/java/dataflow/internal/ModelExclusions.qll @@ -27,7 +27,7 @@ class TestLibrary extends RefType { /** Holds if the given file is a test file. */ private predicate isInTestFile(File file) { file.getAbsolutePath().matches(["%/test/%", "%/guava-tests/%", "%/guava-testlib/%"]) and - not file.getAbsolutePath().matches("%/ql/test/%") // allows our test cases to work + not file.getAbsolutePath().matches(["%/ql/test/%", "%/ql/automodel/test/%"]) // allows our test cases to work } /** Holds if the given compilation unit's package is a JDK internal. */ From b579ab0694222cf2dd6bcb62bdd27ad5466bca0b Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Fri, 18 Aug 2023 11:12:55 +0200 Subject: [PATCH 023/122] Python: Accept `.expected` change --- .../TemplateInjection.expected | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/python/ql/test/experimental/query-tests/Security/CWE-074-TemplateInjection/TemplateInjection.expected b/python/ql/test/experimental/query-tests/Security/CWE-074-TemplateInjection/TemplateInjection.expected index 188aec4c024..09ce53a90e9 100644 --- a/python/ql/test/experimental/query-tests/Security/CWE-074-TemplateInjection/TemplateInjection.expected +++ b/python/ql/test/experimental/query-tests/Security/CWE-074-TemplateInjection/TemplateInjection.expected @@ -4,6 +4,15 @@ edges | AirspeedSsti.py:10:16:10:22 | ControlFlowNode for request | AirspeedSsti.py:10:16:10:27 | ControlFlowNode for Attribute | | AirspeedSsti.py:10:16:10:27 | ControlFlowNode for Attribute | AirspeedSsti.py:10:16:10:43 | ControlFlowNode for Attribute() | | AirspeedSsti.py:10:16:10:43 | ControlFlowNode for Attribute() | AirspeedSsti.py:11:30:11:37 | ControlFlowNode for template | +| CheetahSinks.py:1:26:1:32 | ControlFlowNode for ImportMember | CheetahSinks.py:1:26:1:32 | GSSA Variable request | +| CheetahSinks.py:1:26:1:32 | GSSA Variable request | CheetahSinks.py:10:16:10:22 | ControlFlowNode for request | +| CheetahSinks.py:1:26:1:32 | GSSA Variable request | CheetahSinks.py:21:16:21:22 | ControlFlowNode for request | +| CheetahSinks.py:10:16:10:22 | ControlFlowNode for request | CheetahSinks.py:10:16:10:27 | ControlFlowNode for Attribute | +| CheetahSinks.py:10:16:10:27 | ControlFlowNode for Attribute | CheetahSinks.py:10:16:10:43 | ControlFlowNode for Attribute() | +| CheetahSinks.py:10:16:10:43 | ControlFlowNode for Attribute() | CheetahSinks.py:11:21:11:28 | ControlFlowNode for template | +| CheetahSinks.py:21:16:21:22 | ControlFlowNode for request | CheetahSinks.py:21:16:21:27 | ControlFlowNode for Attribute | +| CheetahSinks.py:21:16:21:27 | ControlFlowNode for Attribute | CheetahSinks.py:21:16:21:43 | ControlFlowNode for Attribute() | +| CheetahSinks.py:21:16:21:43 | ControlFlowNode for Attribute() | CheetahSinks.py:22:20:22:27 | ControlFlowNode for template | | ChevronSsti.py:1:26:1:32 | ControlFlowNode for ImportMember | ChevronSsti.py:1:26:1:32 | GSSA Variable request | | ChevronSsti.py:1:26:1:32 | GSSA Variable request | ChevronSsti.py:10:16:10:22 | ControlFlowNode for request | | ChevronSsti.py:10:16:10:22 | ControlFlowNode for request | ChevronSsti.py:10:16:10:27 | ControlFlowNode for Attribute | @@ -40,6 +49,16 @@ nodes | AirspeedSsti.py:10:16:10:27 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute | | AirspeedSsti.py:10:16:10:43 | ControlFlowNode for Attribute() | semmle.label | ControlFlowNode for Attribute() | | AirspeedSsti.py:11:30:11:37 | ControlFlowNode for template | semmle.label | ControlFlowNode for template | +| CheetahSinks.py:1:26:1:32 | ControlFlowNode for ImportMember | semmle.label | ControlFlowNode for ImportMember | +| CheetahSinks.py:1:26:1:32 | GSSA Variable request | semmle.label | GSSA Variable request | +| CheetahSinks.py:10:16:10:22 | ControlFlowNode for request | semmle.label | ControlFlowNode for request | +| CheetahSinks.py:10:16:10:27 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute | +| CheetahSinks.py:10:16:10:43 | ControlFlowNode for Attribute() | semmle.label | ControlFlowNode for Attribute() | +| CheetahSinks.py:11:21:11:28 | ControlFlowNode for template | semmle.label | ControlFlowNode for template | +| CheetahSinks.py:21:16:21:22 | ControlFlowNode for request | semmle.label | ControlFlowNode for request | +| CheetahSinks.py:21:16:21:27 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute | +| CheetahSinks.py:21:16:21:43 | ControlFlowNode for Attribute() | semmle.label | ControlFlowNode for Attribute() | +| CheetahSinks.py:22:20:22:27 | ControlFlowNode for template | semmle.label | ControlFlowNode for template | | ChevronSsti.py:1:26:1:32 | ControlFlowNode for ImportMember | semmle.label | ControlFlowNode for ImportMember | | ChevronSsti.py:1:26:1:32 | GSSA Variable request | semmle.label | GSSA Variable request | | ChevronSsti.py:10:16:10:22 | ControlFlowNode for request | semmle.label | ControlFlowNode for request | @@ -78,6 +97,8 @@ nodes subpaths #select | AirspeedSsti.py:11:30:11:37 | ControlFlowNode for template | AirspeedSsti.py:2:26:2:32 | ControlFlowNode for ImportMember | AirspeedSsti.py:11:30:11:37 | ControlFlowNode for template | This Template depends on $@. | AirspeedSsti.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | +| CheetahSinks.py:11:21:11:28 | ControlFlowNode for template | CheetahSinks.py:1:26:1:32 | ControlFlowNode for ImportMember | CheetahSinks.py:11:21:11:28 | ControlFlowNode for template | This Template depends on $@. | CheetahSinks.py:1:26:1:32 | ControlFlowNode for ImportMember | user-provided value | +| CheetahSinks.py:22:20:22:27 | ControlFlowNode for template | CheetahSinks.py:1:26:1:32 | ControlFlowNode for ImportMember | CheetahSinks.py:22:20:22:27 | ControlFlowNode for template | This Template depends on $@. | CheetahSinks.py:1:26:1:32 | ControlFlowNode for ImportMember | user-provided value | | ChevronSsti.py:11:27:11:34 | ControlFlowNode for template | ChevronSsti.py:1:26:1:32 | ControlFlowNode for ImportMember | ChevronSsti.py:11:27:11:34 | ControlFlowNode for template | This Template depends on $@. | ChevronSsti.py:1:26:1:32 | ControlFlowNode for ImportMember | user-provided value | | DjangoTemplates.py:9:18:9:25 | ControlFlowNode for template | DjangoTemplates.py:6:8:6:14 | ControlFlowNode for request | DjangoTemplates.py:9:18:9:25 | ControlFlowNode for template | This Template depends on $@. | DjangoTemplates.py:6:8:6:14 | ControlFlowNode for request | user-provided value | | FlaskTemplate.py:11:39:11:66 | ControlFlowNode for Attribute() | FlaskTemplate.py:1:26:1:32 | ControlFlowNode for ImportMember | FlaskTemplate.py:11:39:11:66 | ControlFlowNode for Attribute() | This Template depends on $@. | FlaskTemplate.py:1:26:1:32 | ControlFlowNode for ImportMember | user-provided value | From 4daabdae2be64cbc6bb71b36578799c80c88853b Mon Sep 17 00:00:00 2001 From: Mathias Vorreiter Pedersen Date: Mon, 21 Aug 2023 09:53:40 +0100 Subject: [PATCH 024/122] C++: Promote 'cpp/invalid-pointer-deref' out of experimental. --- .../Security/CWE/CWE-193/InvalidPointerDeref.cpp | 0 .../Security/CWE/CWE-193/InvalidPointerDeref.qhelp | 0 .../Security/CWE/CWE-193/InvalidPointerDeref.ql | 0 .../Security/CWE/CWE-193/pointer-deref/InvalidPointerDeref.qlref | 1 - .../Security/CWE/CWE-193}/AllocationToInvalidPointer.expected | 0 .../Security/CWE/CWE-193}/AllocationToInvalidPointer.ql | 0 .../Security/CWE/CWE-193}/InvalidPointerDeref.expected | 0 .../query-tests/Security/CWE/CWE-193/InvalidPointerDeref.qlref | 1 + .../Security/CWE/CWE-193}/InvalidPointerToDereference.expected | 0 .../Security/CWE/CWE-193}/InvalidPointerToDereference.ql | 0 .../pointer-deref => query-tests/Security/CWE/CWE-193}/test.cpp | 0 11 files changed, 1 insertion(+), 1 deletion(-) rename cpp/ql/src/{experimental => }/Security/CWE/CWE-193/InvalidPointerDeref.cpp (100%) rename cpp/ql/src/{experimental => }/Security/CWE/CWE-193/InvalidPointerDeref.qhelp (100%) rename cpp/ql/src/{experimental => }/Security/CWE/CWE-193/InvalidPointerDeref.ql (100%) delete mode 100644 cpp/ql/test/experimental/query-tests/Security/CWE/CWE-193/pointer-deref/InvalidPointerDeref.qlref rename cpp/ql/test/{experimental/query-tests/Security/CWE/CWE-193/pointer-deref => query-tests/Security/CWE/CWE-193}/AllocationToInvalidPointer.expected (100%) rename cpp/ql/test/{experimental/query-tests/Security/CWE/CWE-193/pointer-deref => query-tests/Security/CWE/CWE-193}/AllocationToInvalidPointer.ql (100%) rename cpp/ql/test/{experimental/query-tests/Security/CWE/CWE-193/pointer-deref => query-tests/Security/CWE/CWE-193}/InvalidPointerDeref.expected (100%) create mode 100644 cpp/ql/test/query-tests/Security/CWE/CWE-193/InvalidPointerDeref.qlref rename cpp/ql/test/{experimental/query-tests/Security/CWE/CWE-193/pointer-deref => query-tests/Security/CWE/CWE-193}/InvalidPointerToDereference.expected (100%) rename cpp/ql/test/{experimental/query-tests/Security/CWE/CWE-193/pointer-deref => query-tests/Security/CWE/CWE-193}/InvalidPointerToDereference.ql (100%) rename cpp/ql/test/{experimental/query-tests/Security/CWE/CWE-193/pointer-deref => query-tests/Security/CWE/CWE-193}/test.cpp (100%) diff --git a/cpp/ql/src/experimental/Security/CWE/CWE-193/InvalidPointerDeref.cpp b/cpp/ql/src/Security/CWE/CWE-193/InvalidPointerDeref.cpp similarity index 100% rename from cpp/ql/src/experimental/Security/CWE/CWE-193/InvalidPointerDeref.cpp rename to cpp/ql/src/Security/CWE/CWE-193/InvalidPointerDeref.cpp diff --git a/cpp/ql/src/experimental/Security/CWE/CWE-193/InvalidPointerDeref.qhelp b/cpp/ql/src/Security/CWE/CWE-193/InvalidPointerDeref.qhelp similarity index 100% rename from cpp/ql/src/experimental/Security/CWE/CWE-193/InvalidPointerDeref.qhelp rename to cpp/ql/src/Security/CWE/CWE-193/InvalidPointerDeref.qhelp diff --git a/cpp/ql/src/experimental/Security/CWE/CWE-193/InvalidPointerDeref.ql b/cpp/ql/src/Security/CWE/CWE-193/InvalidPointerDeref.ql similarity index 100% rename from cpp/ql/src/experimental/Security/CWE/CWE-193/InvalidPointerDeref.ql rename to cpp/ql/src/Security/CWE/CWE-193/InvalidPointerDeref.ql diff --git a/cpp/ql/test/experimental/query-tests/Security/CWE/CWE-193/pointer-deref/InvalidPointerDeref.qlref b/cpp/ql/test/experimental/query-tests/Security/CWE/CWE-193/pointer-deref/InvalidPointerDeref.qlref deleted file mode 100644 index 76da29dc7a0..00000000000 --- a/cpp/ql/test/experimental/query-tests/Security/CWE/CWE-193/pointer-deref/InvalidPointerDeref.qlref +++ /dev/null @@ -1 +0,0 @@ -experimental/Security/CWE/CWE-193/InvalidPointerDeref.ql diff --git a/cpp/ql/test/experimental/query-tests/Security/CWE/CWE-193/pointer-deref/AllocationToInvalidPointer.expected b/cpp/ql/test/query-tests/Security/CWE/CWE-193/AllocationToInvalidPointer.expected similarity index 100% rename from cpp/ql/test/experimental/query-tests/Security/CWE/CWE-193/pointer-deref/AllocationToInvalidPointer.expected rename to cpp/ql/test/query-tests/Security/CWE/CWE-193/AllocationToInvalidPointer.expected diff --git a/cpp/ql/test/experimental/query-tests/Security/CWE/CWE-193/pointer-deref/AllocationToInvalidPointer.ql b/cpp/ql/test/query-tests/Security/CWE/CWE-193/AllocationToInvalidPointer.ql similarity index 100% rename from cpp/ql/test/experimental/query-tests/Security/CWE/CWE-193/pointer-deref/AllocationToInvalidPointer.ql rename to cpp/ql/test/query-tests/Security/CWE/CWE-193/AllocationToInvalidPointer.ql diff --git a/cpp/ql/test/experimental/query-tests/Security/CWE/CWE-193/pointer-deref/InvalidPointerDeref.expected b/cpp/ql/test/query-tests/Security/CWE/CWE-193/InvalidPointerDeref.expected similarity index 100% rename from cpp/ql/test/experimental/query-tests/Security/CWE/CWE-193/pointer-deref/InvalidPointerDeref.expected rename to cpp/ql/test/query-tests/Security/CWE/CWE-193/InvalidPointerDeref.expected diff --git a/cpp/ql/test/query-tests/Security/CWE/CWE-193/InvalidPointerDeref.qlref b/cpp/ql/test/query-tests/Security/CWE/CWE-193/InvalidPointerDeref.qlref new file mode 100644 index 00000000000..b899b6eeb20 --- /dev/null +++ b/cpp/ql/test/query-tests/Security/CWE/CWE-193/InvalidPointerDeref.qlref @@ -0,0 +1 @@ +Security/CWE/CWE-193/InvalidPointerDeref.ql diff --git a/cpp/ql/test/experimental/query-tests/Security/CWE/CWE-193/pointer-deref/InvalidPointerToDereference.expected b/cpp/ql/test/query-tests/Security/CWE/CWE-193/InvalidPointerToDereference.expected similarity index 100% rename from cpp/ql/test/experimental/query-tests/Security/CWE/CWE-193/pointer-deref/InvalidPointerToDereference.expected rename to cpp/ql/test/query-tests/Security/CWE/CWE-193/InvalidPointerToDereference.expected diff --git a/cpp/ql/test/experimental/query-tests/Security/CWE/CWE-193/pointer-deref/InvalidPointerToDereference.ql b/cpp/ql/test/query-tests/Security/CWE/CWE-193/InvalidPointerToDereference.ql similarity index 100% rename from cpp/ql/test/experimental/query-tests/Security/CWE/CWE-193/pointer-deref/InvalidPointerToDereference.ql rename to cpp/ql/test/query-tests/Security/CWE/CWE-193/InvalidPointerToDereference.ql diff --git a/cpp/ql/test/experimental/query-tests/Security/CWE/CWE-193/pointer-deref/test.cpp b/cpp/ql/test/query-tests/Security/CWE/CWE-193/test.cpp similarity index 100% rename from cpp/ql/test/experimental/query-tests/Security/CWE/CWE-193/pointer-deref/test.cpp rename to cpp/ql/test/query-tests/Security/CWE/CWE-193/test.cpp From 70fdfc2ae3010c7a9cefffcd7ad7a366415bb3b8 Mon Sep 17 00:00:00 2001 From: Mathias Vorreiter Pedersen Date: Mon, 21 Aug 2023 09:54:48 +0100 Subject: [PATCH 025/122] C++: Set precision to medium and add security severity. --- cpp/ql/src/Security/CWE/CWE-193/InvalidPointerDeref.ql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cpp/ql/src/Security/CWE/CWE-193/InvalidPointerDeref.ql b/cpp/ql/src/Security/CWE/CWE-193/InvalidPointerDeref.ql index f778b4f776f..943ee61fc7e 100644 --- a/cpp/ql/src/Security/CWE/CWE-193/InvalidPointerDeref.ql +++ b/cpp/ql/src/Security/CWE/CWE-193/InvalidPointerDeref.ql @@ -4,7 +4,8 @@ * and may lead to security vulnerabilities. * @kind path-problem * @problem.severity error - * @precision high + * @security-severity 9.3 + * @precision medium * @id cpp/invalid-pointer-deref * @tags reliability * security From 0a41acc0a6dacb8c3b3c73b5d525cfe37f760749 Mon Sep 17 00:00:00 2001 From: Mathias Vorreiter Pedersen Date: Mon, 21 Aug 2023 10:17:28 +0100 Subject: [PATCH 026/122] C++: Add change note. --- cpp/ql/src/change-notes/2023-08-21-invalid-pointer-deref.md | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 cpp/ql/src/change-notes/2023-08-21-invalid-pointer-deref.md diff --git a/cpp/ql/src/change-notes/2023-08-21-invalid-pointer-deref.md b/cpp/ql/src/change-notes/2023-08-21-invalid-pointer-deref.md new file mode 100644 index 00000000000..d8207a75604 --- /dev/null +++ b/cpp/ql/src/change-notes/2023-08-21-invalid-pointer-deref.md @@ -0,0 +1,4 @@ +--- +category: newQuery +--- +* Added a new query, `cpp/invalid-pointer-deref`, to detect out-of-bounds pointer reads and writes. From e776178be52a09b3eb194a0b05bd91218dccfe67 Mon Sep 17 00:00:00 2001 From: Mathias Vorreiter Pedersen Date: Mon, 21 Aug 2023 10:07:39 +0100 Subject: [PATCH 027/122] C++: Add some whitespace to make stuff appear in the diff. --- .../CWE/CWE-193/InvalidPointerDeref.qhelp | 50 +-- .../CWE/CWE-193/InvalidPointerDeref.ql | 284 +++++++++--------- 2 files changed, 167 insertions(+), 167 deletions(-) diff --git a/cpp/ql/src/Security/CWE/CWE-193/InvalidPointerDeref.qhelp b/cpp/ql/src/Security/CWE/CWE-193/InvalidPointerDeref.qhelp index 4f590659112..db249eb01eb 100644 --- a/cpp/ql/src/Security/CWE/CWE-193/InvalidPointerDeref.qhelp +++ b/cpp/ql/src/Security/CWE/CWE-193/InvalidPointerDeref.qhelp @@ -1,31 +1,31 @@ - - - -

The program performs an out-of-bounds read or write operation. In addition to causing program instability, techniques exist which may allow an attacker to use this vulnerability to execute arbitrary code.

+ + + +

The program performs an out-of-bounds read or write operation. In addition to causing program instability, techniques exist which may allow an attacker to use this vulnerability to execute arbitrary code.

- - + + -

Ensure that pointer dereferences are properly guarded to ensure that they cannot be used to read or write past the end of the allocation.

+

Ensure that pointer dereferences are properly guarded to ensure that they cannot be used to read or write past the end of the allocation.

- - -

The first example allocates a buffer of size size and creates a local variable that stores the location that is one byte past the end of the allocation. -This local variable is then dereferenced which results in an out-of-bounds write. -The second example subtracts one from the end variable before dereferencing it. This subtraction ensures that the write correctly updates the final byte of the allocation.

- + + +

The first example allocates a buffer of size size and creates a local variable that stores the location that is one byte past the end of the allocation. + This local variable is then dereferenced which results in an out-of-bounds write. + The second example subtracts one from the end variable before dereferencing it. This subtraction ensures that the write correctly updates the final byte of the allocation.

+ - - + + -
  • CERT C Coding Standard: -ARR30-C. Do not form or use out-of-bounds pointers or array subscripts.
    • -
  • -OWASP: -Buffer Overflow. -
    • +
  • CERT C Coding Standard: + ARR30-C. Do not form or use out-of-bounds pointers or array subscripts.
    • +
  • + OWASP: + Buffer Overflow. +
    • -     -     + + diff --git a/cpp/ql/src/Security/CWE/CWE-193/InvalidPointerDeref.ql b/cpp/ql/src/Security/CWE/CWE-193/InvalidPointerDeref.ql index 943ee61fc7e..df7e87d0877 100644 --- a/cpp/ql/src/Security/CWE/CWE-193/InvalidPointerDeref.ql +++ b/cpp/ql/src/Security/CWE/CWE-193/InvalidPointerDeref.ql @@ -1,153 +1,153 @@ -/** - * @name Invalid pointer dereference - * @description Dereferencing a pointer that points past it allocation is undefined behavior - * and may lead to security vulnerabilities. - * @kind path-problem - * @problem.severity error - * @security-severity 9.3 - * @precision medium - * @id cpp/invalid-pointer-deref - * @tags reliability - * security - * experimental - * external/cwe/cwe-119 - * external/cwe/cwe-125 - * external/cwe/cwe-193 - * external/cwe/cwe-787 - */ + /** + * @name Invalid pointer dereference + * @description Dereferencing a pointer that points past it allocation is undefined behavior + * and may lead to security vulnerabilities. + * @kind path-problem + * @problem.severity error + * @security-severity 9.3 + * @precision medium + * @id cpp/invalid-pointer-deref + * @tags reliability + * security + * experimental + * external/cwe/cwe-119 + * external/cwe/cwe-125 + * external/cwe/cwe-193 + * external/cwe/cwe-787 + */ -/* - * High-level description of the query: - * - * The goal of this query is to identify issues such as: - * ```cpp - * 1. int* base = new int[size]; - * 2. int* end = base + size; - * 3. for(int* p = base; p <= end; ++p) { - * 4. *p = 0; // BUG: Should have been bounded by `p < end`. - * 5. } - * ``` - * In order to do this, we split the problem into three subtasks: - * 1. First, we find flow from `new int[size]` to `base + size`. - * 2. Then, we find flow from `base + size` to `end` (on line 3). - * 3. Finally, we use range-analysis to find a write to (or read from) a pointer that may be greater than or equal to `end`. - * - * Step 1 is implemented in `AllocationToInvalidPointer.qll`, and step 2 is implemented by - * `InvalidPointerToDereference.qll`. See those files for the description of these. - * - * This file imports both libraries and defines a final dataflow configuration that constructs the full path from - * the allocation to the dereference of the out-of-bounds pointer. This is done for several reasons: - * 1. It means the user is able to inspect the entire path from the allocation to the dereference, which can be useful - * to understand the problem highlighted. - * 2. It ensures that the call-contexts line up correctly when we transition from step 1 to step 2. See the - * `test_missing_call_context_1` and `test_missing_call_context_2` tests for how this may flag false positives - * without this final configuration. - * - * The source of the final path is an allocation that is: - * 1. identified as flowing to an invalid pointer (by `AllocationToInvalidPointer`), and - * 2. for which the invalid pointer flows to a dereference (as identified by `InvalidPointerToDereference`). - * - * The path can be described in 3 "chunks": - * 1. One path from the allocation to the construction of the invalid pointer - * 2. Another path from the construction of the invalid pointer to the final pointer that is about to be dereferenced. - * 3. Finally, a single step from the dataflow node that represents the final pointer to the dereference. - * - * Step 1 happens when the flow state is `TInitial`, and step 2 and 3 happen when the flow state is `TPointerArith(pai)` - * where the pointer-arithmetic instruction `pai` tracks the instruction that generated the out-of-bounds pointer. This - * instruction is used in the construction of the alert message. - * - * The set of pointer-arithmetic instructions that define the `TPointerArith` flow state is restricted to be the pointer- - * arithmetic instructions that both receive flow from the allocation (as identified by `AllocationToInvalidPointer.qll`), - * and further flow to a dereference (as identified by `InvalidPointerToDereference.qll`). - */ + /* + * High-level description of the query: + * + * The goal of this query is to identify issues such as: + * ```cpp + * 1. int* base = new int[size]; + * 2. int* end = base + size; + * 3. for(int* p = base; p <= end; ++p) { + * 4. *p = 0; // BUG: Should have been bounded by `p < end`. + * 5. } + * ``` + * In order to do this, we split the problem into three subtasks: + * 1. First, we find flow from `new int[size]` to `base + size`. + * 2. Then, we find flow from `base + size` to `end` (on line 3). + * 3. Finally, we use range-analysis to find a write to (or read from) a pointer that may be greater than or equal to `end`. + * + * Step 1 is implemented in `AllocationToInvalidPointer.qll`, and step 2 is implemented by + * `InvalidPointerToDereference.qll`. See those files for the description of these. + * + * This file imports both libraries and defines a final dataflow configuration that constructs the full path from + * the allocation to the dereference of the out-of-bounds pointer. This is done for several reasons: + * 1. It means the user is able to inspect the entire path from the allocation to the dereference, which can be useful + * to understand the problem highlighted. + * 2. It ensures that the call-contexts line up correctly when we transition from step 1 to step 2. See the + * `test_missing_call_context_1` and `test_missing_call_context_2` tests for how this may flag false positives + * without this final configuration. + * + * The source of the final path is an allocation that is: + * 1. identified as flowing to an invalid pointer (by `AllocationToInvalidPointer`), and + * 2. for which the invalid pointer flows to a dereference (as identified by `InvalidPointerToDereference`). + * + * The path can be described in 3 "chunks": + * 1. One path from the allocation to the construction of the invalid pointer + * 2. Another path from the construction of the invalid pointer to the final pointer that is about to be dereferenced. + * 3. Finally, a single step from the dataflow node that represents the final pointer to the dereference. + * + * Step 1 happens when the flow state is `TInitial`, and step 2 and 3 happen when the flow state is `TPointerArith(pai)` + * where the pointer-arithmetic instruction `pai` tracks the instruction that generated the out-of-bounds pointer. This + * instruction is used in the construction of the alert message. + * + * The set of pointer-arithmetic instructions that define the `TPointerArith` flow state is restricted to be the pointer- + * arithmetic instructions that both receive flow from the allocation (as identified by `AllocationToInvalidPointer.qll`), + * and further flow to a dereference (as identified by `InvalidPointerToDereference.qll`). + */ -import cpp -import semmle.code.cpp.dataflow.new.DataFlow -import semmle.code.cpp.ir.IR -import FinalFlow::PathGraph -import semmle.code.cpp.security.InvalidPointerDereference.AllocationToInvalidPointer -import semmle.code.cpp.security.InvalidPointerDereference.InvalidPointerToDereference + import cpp + import semmle.code.cpp.dataflow.new.DataFlow + import semmle.code.cpp.ir.IR + import FinalFlow::PathGraph + import semmle.code.cpp.security.InvalidPointerDereference.AllocationToInvalidPointer + import semmle.code.cpp.security.InvalidPointerDereference.InvalidPointerToDereference -/** - * A configuration that represents the full dataflow path all the way from - * the allocation to the dereference. We need this final dataflow traversal - * to ensure that the transition from the sink in `AllocToInvalidPointerConfig` - * to the source in `InvalidPointerToDerefFlow` did not make us construct an - * infeasible path (which can happen since the transition from one configuration - * to the next does not preserve information about call contexts). - */ -module FinalConfig implements DataFlow::StateConfigSig { - newtype FlowState = - additional TInitial() or - additional TPointerArith(PointerArithmeticInstruction pai) { - operationIsOffBy(_, pai, _, _, _, _, _) + /** + * A configuration that represents the full dataflow path all the way from + * the allocation to the dereference. We need this final dataflow traversal + * to ensure that the transition from the sink in `AllocToInvalidPointerConfig` + * to the source in `InvalidPointerToDerefFlow` did not make us construct an + * infeasible path (which can happen since the transition from one configuration + * to the next does not preserve information about call contexts). + */ + module FinalConfig implements DataFlow::StateConfigSig { + newtype FlowState = + additional TInitial() or + additional TPointerArith(PointerArithmeticInstruction pai) { + operationIsOffBy(_, pai, _, _, _, _, _) + } + + predicate isSource(DataFlow::Node source, FlowState state) { + state = TInitial() and + operationIsOffBy(source, _, _, _, _, _, _) } - predicate isSource(DataFlow::Node source, FlowState state) { - state = TInitial() and - operationIsOffBy(source, _, _, _, _, _, _) + predicate isSink(DataFlow::Node sink, FlowState state) { + exists(PointerArithmeticInstruction pai | + operationIsOffBy(_, pai, _, _, _, sink, _) and + state = TPointerArith(pai) + ) + } + + predicate isAdditionalFlowStep( + DataFlow::Node node1, FlowState state1, DataFlow::Node node2, FlowState state2 + ) { + // A step from the left-hand side of a pointer-arithmetic operation that has been + // identified as creating an out-of-bounds pointer to the result of the pointer-arithmetic + // operation. + exists(PointerArithmeticInstruction pai | + pointerAddInstructionHasBounds(_, pai, node1, _) and + operationIsOffBy(_, pai, node2, _, _, _, _) and + state1 = TInitial() and + state2 = TPointerArith(pai) + ) + or + // A step from an out-of-bounds address to the operation (which is either a `StoreInstruction` + // or a `LoadInstruction`) that dereferences the address. + // This step exists purely for aesthetic reasons: we want the alert to be placed at the operation + // that causes the dereference, and not at the address that flows into the operation. + state1 = state2 and + exists(PointerArithmeticInstruction pai | + state1 = TPointerArith(pai) and + operationIsOffBy(_, pai, _, node1, _, node2, _) + ) + } } - predicate isSink(DataFlow::Node sink, FlowState state) { - exists(PointerArithmeticInstruction pai | - operationIsOffBy(_, pai, _, _, _, sink, _) and - state = TPointerArith(pai) - ) - } + module FinalFlow = DataFlow::GlobalWithState; - predicate isAdditionalFlowStep( - DataFlow::Node node1, FlowState state1, DataFlow::Node node2, FlowState state2 + /** + * Holds if `source` is an allocation that flows into the left-hand side of `pai`, which produces an out-of-bounds + * pointer that flows into an address that is dereferenced by `sink` (which is either a `LoadInstruction` or a + * `StoreInstruction`). The end result is that `sink` writes to an address that is off-by-`delta` from the end of + * the allocation. The string `operation` describes whether the `sink` is a load or a store (which is then used + * to produce the alert message). + * + * Note that multiple `delta`s can exist for a given `(source, pai, sink)` triplet. + */ + predicate hasFlowPath( + FinalFlow::PathNode source, FinalFlow::PathNode sink, PointerArithmeticInstruction pai, + string operation, int delta ) { - // A step from the left-hand side of a pointer-arithmetic operation that has been - // identified as creating an out-of-bounds pointer to the result of the pointer-arithmetic - // operation. - exists(PointerArithmeticInstruction pai | - pointerAddInstructionHasBounds(_, pai, node1, _) and - operationIsOffBy(_, pai, node2, _, _, _, _) and - state1 = TInitial() and - state2 = TPointerArith(pai) - ) - or - // A step from an out-of-bounds address to the operation (which is either a `StoreInstruction` - // or a `LoadInstruction`) that dereferences the address. - // This step exists purely for aesthetic reasons: we want the alert to be placed at the operation - // that causes the dereference, and not at the address that flows into the operation. - state1 = state2 and - exists(PointerArithmeticInstruction pai | - state1 = TPointerArith(pai) and - operationIsOffBy(_, pai, _, node1, _, node2, _) - ) + FinalFlow::flowPath(source, sink) and + operationIsOffBy(source.getNode(), pai, _, _, operation, sink.getNode(), delta) and + sink.getState() = FinalConfig::TPointerArith(pai) } -} -module FinalFlow = DataFlow::GlobalWithState; - -/** - * Holds if `source` is an allocation that flows into the left-hand side of `pai`, which produces an out-of-bounds - * pointer that flows into an address that is dereferenced by `sink` (which is either a `LoadInstruction` or a - * `StoreInstruction`). The end result is that `sink` writes to an address that is off-by-`delta` from the end of - * the allocation. The string `operation` describes whether the `sink` is a load or a store (which is then used - * to produce the alert message). - * - * Note that multiple `delta`s can exist for a given `(source, pai, sink)` triplet. - */ -predicate hasFlowPath( - FinalFlow::PathNode source, FinalFlow::PathNode sink, PointerArithmeticInstruction pai, - string operation, int delta -) { - FinalFlow::flowPath(source, sink) and - operationIsOffBy(source.getNode(), pai, _, _, operation, sink.getNode(), delta) and - sink.getState() = FinalConfig::TPointerArith(pai) -} - -from - FinalFlow::PathNode source, FinalFlow::PathNode sink, int k, string kstr, - PointerArithmeticInstruction pai, string operation, Expr offset, DataFlow::Node n -where - k = min(int cand | hasFlowPath(source, sink, pai, operation, cand)) and - offset = pai.getRight().getUnconvertedResultExpression() and - n = source.getNode() and - if k = 0 then kstr = "" else kstr = " + " + k -select sink.getNode(), source, sink, - "This " + operation + " might be out of bounds, as the pointer might be equal to $@ + $@" + kstr + - ".", n, n.toString(), offset, offset.toString() + from + FinalFlow::PathNode source, FinalFlow::PathNode sink, int k, string kstr, + PointerArithmeticInstruction pai, string operation, Expr offset, DataFlow::Node n + where + k = min(int cand | hasFlowPath(source, sink, pai, operation, cand)) and + offset = pai.getRight().getUnconvertedResultExpression() and + n = source.getNode() and + if k = 0 then kstr = "" else kstr = " + " + k + select sink.getNode(), source, sink, + "This " + operation + " might be out of bounds, as the pointer might be equal to $@ + $@" + kstr + + ".", n, n.toString(), offset, offset.toString() From 3aeacf6df34a0d50ffb324cfd6dc6660428296c1 Mon Sep 17 00:00:00 2001 From: Anders Starcke Henriksen Date: Tue, 22 Aug 2023 09:37:31 +0200 Subject: [PATCH 028/122] Update publish script to have right path. --- java/ql/automodel/publish.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/ql/automodel/publish.sh b/java/ql/automodel/publish.sh index 202cdd690a7..b13570c950f 100755 --- a/java/ql/automodel/publish.sh +++ b/java/ql/automodel/publish.sh @@ -2,7 +2,7 @@ set -e AUTOMODEL_ROOT="$(readlink -f "$(dirname $0)")" -WORKSPACE_ROOT="$AUTOMODEL_ROOT/../../../.." +WORKSPACE_ROOT="$AUTOMODEL_ROOT/../../.." GRPS="automodel,-test" if [ -z "$CODEQL_DIST" ]; then From 3b8b33a94de439776a32ed0f29b012500511d0ee Mon Sep 17 00:00:00 2001 From: Anders Starcke Henriksen Date: Tue, 22 Aug 2023 09:45:52 +0200 Subject: [PATCH 029/122] Released pack. --- java/ql/automodel/src/qlpack.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/ql/automodel/src/qlpack.yml b/java/ql/automodel/src/qlpack.yml index 1ae0225cb2a..851dbe69e82 100644 --- a/java/ql/automodel/src/qlpack.yml +++ b/java/ql/automodel/src/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/java-automodel-queries -version: 0.0.1-dev +version: 0.0.3-dev groups: - java - automodel From abe28cb106e14d228a1e55ec216e6d1957afe4f6 Mon Sep 17 00:00:00 2001 From: Mathias Vorreiter Pedersen Date: Tue, 22 Aug 2023 13:02:29 +0100 Subject: [PATCH 030/122] Update cpp/ql/src/Security/CWE/CWE-193/InvalidPointerDeref.ql Co-authored-by: mc <42146119+mchammer01@users.noreply.github.com> --- cpp/ql/src/Security/CWE/CWE-193/InvalidPointerDeref.ql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/ql/src/Security/CWE/CWE-193/InvalidPointerDeref.ql b/cpp/ql/src/Security/CWE/CWE-193/InvalidPointerDeref.ql index df7e87d0877..f78b377a730 100644 --- a/cpp/ql/src/Security/CWE/CWE-193/InvalidPointerDeref.ql +++ b/cpp/ql/src/Security/CWE/CWE-193/InvalidPointerDeref.ql @@ -1,6 +1,6 @@ /** * @name Invalid pointer dereference - * @description Dereferencing a pointer that points past it allocation is undefined behavior + * @description Dereferencing a pointer that points past its allocation is undefined behavior * and may lead to security vulnerabilities. * @kind path-problem * @problem.severity error From e88277bd3bc8a09c9cbb38bfb3842daca0511084 Mon Sep 17 00:00:00 2001 From: Mathias Vorreiter Pedersen Date: Tue, 22 Aug 2023 13:02:37 +0100 Subject: [PATCH 031/122] Update cpp/ql/src/Security/CWE/CWE-193/InvalidPointerDeref.qhelp Co-authored-by: mc <42146119+mchammer01@users.noreply.github.com> --- cpp/ql/src/Security/CWE/CWE-193/InvalidPointerDeref.qhelp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/ql/src/Security/CWE/CWE-193/InvalidPointerDeref.qhelp b/cpp/ql/src/Security/CWE/CWE-193/InvalidPointerDeref.qhelp index db249eb01eb..41e75244778 100644 --- a/cpp/ql/src/Security/CWE/CWE-193/InvalidPointerDeref.qhelp +++ b/cpp/ql/src/Security/CWE/CWE-193/InvalidPointerDeref.qhelp @@ -13,7 +13,7 @@

    The first example allocates a buffer of size size and creates a local variable that stores the location that is one byte past the end of the allocation. - This local variable is then dereferenced which results in an out-of-bounds write. + This local variable is then dereferenced, which results in an out-of-bounds write. The second example subtracts one from the end variable before dereferencing it. This subtraction ensures that the write correctly updates the final byte of the allocation.

    From 1c3a0d163259d9869546c20e841c5a493c394496 Mon Sep 17 00:00:00 2001 From: Mathias Vorreiter Pedersen Date: Tue, 22 Aug 2023 13:03:07 +0100 Subject: [PATCH 032/122] Update cpp/ql/src/Security/CWE/CWE-193/InvalidPointerDeref.qhelp Co-authored-by: mc <42146119+mchammer01@users.noreply.github.com> --- cpp/ql/src/Security/CWE/CWE-193/InvalidPointerDeref.qhelp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/ql/src/Security/CWE/CWE-193/InvalidPointerDeref.qhelp b/cpp/ql/src/Security/CWE/CWE-193/InvalidPointerDeref.qhelp index 41e75244778..46696c89823 100644 --- a/cpp/ql/src/Security/CWE/CWE-193/InvalidPointerDeref.qhelp +++ b/cpp/ql/src/Security/CWE/CWE-193/InvalidPointerDeref.qhelp @@ -3,7 +3,7 @@ "qhelp.dtd"> -

    The program performs an out-of-bounds read or write operation. In addition to causing program instability, techniques exist which may allow an attacker to use this vulnerability to execute arbitrary code.

    +

    The program performs an out-of-bounds read or write operation, which can cause program instability. In addition, attackers may take advantage of the situation, and implement techniques to use this vulnerability to execute arbitrary code.

     From bbce7ee96de517ed1cbfcbad4b19fbb784a275a6 Mon Sep 17 00:00:00 2001 From: Mathias Vorreiter Pedersen Date: Tue, 22 Aug 2023 13:37:24 +0100 Subject: [PATCH 033/122] C++: Cache 'getAnInput' since it's now used in several queries. --- cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowUtil.qll | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowUtil.qll b/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowUtil.qll index 6470741d541..42198d12372 100644 --- a/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowUtil.qll +++ b/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowUtil.qll @@ -550,6 +550,7 @@ class SsaPhiNode extends Node, TSsaPhiNode { * `fromBackEdge` is true if data flows along a back-edge, * and `false` otherwise. */ + cached final Node getAnInput(boolean fromBackEdge) { localFlowStep(result, this) and if phi.getBasicBlock().dominates(result.getBasicBlock()) From 66f11d427bb20e128fa9a6f01aa6e1179a89a5fc Mon Sep 17 00:00:00 2001 From: Mathias Vorreiter Pedersen Date: Tue, 22 Aug 2023 13:39:38 +0100 Subject: [PATCH 034/122] C++: Simplify description. --- cpp/ql/src/Security/CWE/CWE-193/InvalidPointerDeref.ql | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cpp/ql/src/Security/CWE/CWE-193/InvalidPointerDeref.ql b/cpp/ql/src/Security/CWE/CWE-193/InvalidPointerDeref.ql index f78b377a730..e42c4c43b2c 100644 --- a/cpp/ql/src/Security/CWE/CWE-193/InvalidPointerDeref.ql +++ b/cpp/ql/src/Security/CWE/CWE-193/InvalidPointerDeref.ql @@ -1,7 +1,6 @@ /** * @name Invalid pointer dereference - * @description Dereferencing a pointer that points past its allocation is undefined behavior - * and may lead to security vulnerabilities. + * @description Dereferencing an out-of-bounds pointer is undefined behavior and may lead to security vulnerabilities. * @kind path-problem * @problem.severity error * @security-severity 9.3 From 530c950b4162b4471106475490f50d6ef8b2b083 Mon Sep 17 00:00:00 2001 From: Mathias Vorreiter Pedersen Date: Tue, 22 Aug 2023 13:40:00 +0100 Subject: [PATCH 035/122] C++: Fix formatting. --- .../CWE/CWE-193/InvalidPointerDeref.qhelp | 50 ++-- .../CWE/CWE-193/InvalidPointerDeref.ql | 282 +++++++++--------- 2 files changed, 166 insertions(+), 166 deletions(-) diff --git a/cpp/ql/src/Security/CWE/CWE-193/InvalidPointerDeref.qhelp b/cpp/ql/src/Security/CWE/CWE-193/InvalidPointerDeref.qhelp index 46696c89823..19c0e00215b 100644 --- a/cpp/ql/src/Security/CWE/CWE-193/InvalidPointerDeref.qhelp +++ b/cpp/ql/src/Security/CWE/CWE-193/InvalidPointerDeref.qhelp @@ -1,31 +1,31 @@ - - - -

    The program performs an out-of-bounds read or write operation, which can cause program instability. In addition, attackers may take advantage of the situation, and implement techniques to use this vulnerability to execute arbitrary code.

    + + + +

    The program performs an out-of-bounds read or write operation, which can cause program instability. In addition, attackers may take advantage of the situation, and implement techniques to use this vulnerability to execute arbitrary code.

    -     - +     + -

    Ensure that pointer dereferences are properly guarded to ensure that they cannot be used to read or write past the end of the allocation.

    +

    Ensure that pointer dereferences are properly guarded to ensure that they cannot be used to read or write past the end of the allocation.

    -     - -

    The first example allocates a buffer of size size and creates a local variable that stores the location that is one byte past the end of the allocation. - This local variable is then dereferenced, which results in an out-of-bounds write. - The second example subtracts one from the end variable before dereferencing it. This subtraction ensures that the write correctly updates the final byte of the allocation.

    - +     + +

    The first example allocates a buffer of size size and creates a local variable that stores the location that is one byte past the end of the allocation. +This local variable is then dereferenced, which results in an out-of-bounds write. +The second example subtracts one from the end variable before dereferencing it. This subtraction ensures that the write correctly updates the final byte of the allocation.

    + -     - +     + -
  • CERT C Coding Standard: - ARR30-C. Do not form or use out-of-bounds pointers or array subscripts.
    • -
  • - OWASP: - Buffer Overflow. -
    • +
  • CERT C Coding Standard: +ARR30-C. Do not form or use out-of-bounds pointers or array subscripts.
    • +
  • +OWASP: +Buffer Overflow. +
    • -     - + + diff --git a/cpp/ql/src/Security/CWE/CWE-193/InvalidPointerDeref.ql b/cpp/ql/src/Security/CWE/CWE-193/InvalidPointerDeref.ql index e42c4c43b2c..14e77e116df 100644 --- a/cpp/ql/src/Security/CWE/CWE-193/InvalidPointerDeref.ql +++ b/cpp/ql/src/Security/CWE/CWE-193/InvalidPointerDeref.ql @@ -1,152 +1,152 @@ - /** - * @name Invalid pointer dereference - * @description Dereferencing an out-of-bounds pointer is undefined behavior and may lead to security vulnerabilities. - * @kind path-problem - * @problem.severity error - * @security-severity 9.3 - * @precision medium - * @id cpp/invalid-pointer-deref - * @tags reliability - * security - * experimental - * external/cwe/cwe-119 - * external/cwe/cwe-125 - * external/cwe/cwe-193 - * external/cwe/cwe-787 - */ +/** + * @name Invalid pointer dereference + * @description Dereferencing an out-of-bounds pointer is undefined behavior and may lead to security vulnerabilities. + * @kind path-problem + * @problem.severity error + * @security-severity 9.3 + * @precision medium + * @id cpp/invalid-pointer-deref + * @tags reliability + * security + * experimental + * external/cwe/cwe-119 + * external/cwe/cwe-125 + * external/cwe/cwe-193 + * external/cwe/cwe-787 + */ - /* - * High-level description of the query: - * - * The goal of this query is to identify issues such as: - * ```cpp - * 1. int* base = new int[size]; - * 2. int* end = base + size; - * 3. for(int* p = base; p <= end; ++p) { - * 4. *p = 0; // BUG: Should have been bounded by `p < end`. - * 5. } - * ``` - * In order to do this, we split the problem into three subtasks: - * 1. First, we find flow from `new int[size]` to `base + size`. - * 2. Then, we find flow from `base + size` to `end` (on line 3). - * 3. Finally, we use range-analysis to find a write to (or read from) a pointer that may be greater than or equal to `end`. - * - * Step 1 is implemented in `AllocationToInvalidPointer.qll`, and step 2 is implemented by - * `InvalidPointerToDereference.qll`. See those files for the description of these. - * - * This file imports both libraries and defines a final dataflow configuration that constructs the full path from - * the allocation to the dereference of the out-of-bounds pointer. This is done for several reasons: - * 1. It means the user is able to inspect the entire path from the allocation to the dereference, which can be useful - * to understand the problem highlighted. - * 2. It ensures that the call-contexts line up correctly when we transition from step 1 to step 2. See the - * `test_missing_call_context_1` and `test_missing_call_context_2` tests for how this may flag false positives - * without this final configuration. - * - * The source of the final path is an allocation that is: - * 1. identified as flowing to an invalid pointer (by `AllocationToInvalidPointer`), and - * 2. for which the invalid pointer flows to a dereference (as identified by `InvalidPointerToDereference`). - * - * The path can be described in 3 "chunks": - * 1. One path from the allocation to the construction of the invalid pointer - * 2. Another path from the construction of the invalid pointer to the final pointer that is about to be dereferenced. - * 3. Finally, a single step from the dataflow node that represents the final pointer to the dereference. - * - * Step 1 happens when the flow state is `TInitial`, and step 2 and 3 happen when the flow state is `TPointerArith(pai)` - * where the pointer-arithmetic instruction `pai` tracks the instruction that generated the out-of-bounds pointer. This - * instruction is used in the construction of the alert message. - * - * The set of pointer-arithmetic instructions that define the `TPointerArith` flow state is restricted to be the pointer- - * arithmetic instructions that both receive flow from the allocation (as identified by `AllocationToInvalidPointer.qll`), - * and further flow to a dereference (as identified by `InvalidPointerToDereference.qll`). - */ +/* + * High-level description of the query: + * + * The goal of this query is to identify issues such as: + * ```cpp + * 1. int* base = new int[size]; + * 2. int* end = base + size; + * 3. for(int* p = base; p <= end; ++p) { + * 4. *p = 0; // BUG: Should have been bounded by `p < end`. + * 5. } + * ``` + * In order to do this, we split the problem into three subtasks: + * 1. First, we find flow from `new int[size]` to `base + size`. + * 2. Then, we find flow from `base + size` to `end` (on line 3). + * 3. Finally, we use range-analysis to find a write to (or read from) a pointer that may be greater than or equal to `end`. + * + * Step 1 is implemented in `AllocationToInvalidPointer.qll`, and step 2 is implemented by + * `InvalidPointerToDereference.qll`. See those files for the description of these. + * + * This file imports both libraries and defines a final dataflow configuration that constructs the full path from + * the allocation to the dereference of the out-of-bounds pointer. This is done for several reasons: + * 1. It means the user is able to inspect the entire path from the allocation to the dereference, which can be useful + * to understand the problem highlighted. + * 2. It ensures that the call-contexts line up correctly when we transition from step 1 to step 2. See the + * `test_missing_call_context_1` and `test_missing_call_context_2` tests for how this may flag false positives + * without this final configuration. + * + * The source of the final path is an allocation that is: + * 1. identified as flowing to an invalid pointer (by `AllocationToInvalidPointer`), and + * 2. for which the invalid pointer flows to a dereference (as identified by `InvalidPointerToDereference`). + * + * The path can be described in 3 "chunks": + * 1. One path from the allocation to the construction of the invalid pointer + * 2. Another path from the construction of the invalid pointer to the final pointer that is about to be dereferenced. + * 3. Finally, a single step from the dataflow node that represents the final pointer to the dereference. + * + * Step 1 happens when the flow state is `TInitial`, and step 2 and 3 happen when the flow state is `TPointerArith(pai)` + * where the pointer-arithmetic instruction `pai` tracks the instruction that generated the out-of-bounds pointer. This + * instruction is used in the construction of the alert message. + * + * The set of pointer-arithmetic instructions that define the `TPointerArith` flow state is restricted to be the pointer- + * arithmetic instructions that both receive flow from the allocation (as identified by `AllocationToInvalidPointer.qll`), + * and further flow to a dereference (as identified by `InvalidPointerToDereference.qll`). + */ - import cpp - import semmle.code.cpp.dataflow.new.DataFlow - import semmle.code.cpp.ir.IR - import FinalFlow::PathGraph - import semmle.code.cpp.security.InvalidPointerDereference.AllocationToInvalidPointer - import semmle.code.cpp.security.InvalidPointerDereference.InvalidPointerToDereference +import cpp +import semmle.code.cpp.dataflow.new.DataFlow +import semmle.code.cpp.ir.IR +import FinalFlow::PathGraph +import semmle.code.cpp.security.InvalidPointerDereference.AllocationToInvalidPointer +import semmle.code.cpp.security.InvalidPointerDereference.InvalidPointerToDereference - /** - * A configuration that represents the full dataflow path all the way from - * the allocation to the dereference. We need this final dataflow traversal - * to ensure that the transition from the sink in `AllocToInvalidPointerConfig` - * to the source in `InvalidPointerToDerefFlow` did not make us construct an - * infeasible path (which can happen since the transition from one configuration - * to the next does not preserve information about call contexts). - */ - module FinalConfig implements DataFlow::StateConfigSig { - newtype FlowState = - additional TInitial() or - additional TPointerArith(PointerArithmeticInstruction pai) { - operationIsOffBy(_, pai, _, _, _, _, _) - } - - predicate isSource(DataFlow::Node source, FlowState state) { - state = TInitial() and - operationIsOffBy(source, _, _, _, _, _, _) +/** + * A configuration that represents the full dataflow path all the way from + * the allocation to the dereference. We need this final dataflow traversal + * to ensure that the transition from the sink in `AllocToInvalidPointerConfig` + * to the source in `InvalidPointerToDerefFlow` did not make us construct an + * infeasible path (which can happen since the transition from one configuration + * to the next does not preserve information about call contexts). + */ +module FinalConfig implements DataFlow::StateConfigSig { + newtype FlowState = + additional TInitial() or + additional TPointerArith(PointerArithmeticInstruction pai) { + operationIsOffBy(_, pai, _, _, _, _, _) } - predicate isSink(DataFlow::Node sink, FlowState state) { - exists(PointerArithmeticInstruction pai | - operationIsOffBy(_, pai, _, _, _, sink, _) and - state = TPointerArith(pai) - ) - } - - predicate isAdditionalFlowStep( - DataFlow::Node node1, FlowState state1, DataFlow::Node node2, FlowState state2 - ) { - // A step from the left-hand side of a pointer-arithmetic operation that has been - // identified as creating an out-of-bounds pointer to the result of the pointer-arithmetic - // operation. - exists(PointerArithmeticInstruction pai | - pointerAddInstructionHasBounds(_, pai, node1, _) and - operationIsOffBy(_, pai, node2, _, _, _, _) and - state1 = TInitial() and - state2 = TPointerArith(pai) - ) - or - // A step from an out-of-bounds address to the operation (which is either a `StoreInstruction` - // or a `LoadInstruction`) that dereferences the address. - // This step exists purely for aesthetic reasons: we want the alert to be placed at the operation - // that causes the dereference, and not at the address that flows into the operation. - state1 = state2 and - exists(PointerArithmeticInstruction pai | - state1 = TPointerArith(pai) and - operationIsOffBy(_, pai, _, node1, _, node2, _) - ) - } + predicate isSource(DataFlow::Node source, FlowState state) { + state = TInitial() and + operationIsOffBy(source, _, _, _, _, _, _) } - module FinalFlow = DataFlow::GlobalWithState; + predicate isSink(DataFlow::Node sink, FlowState state) { + exists(PointerArithmeticInstruction pai | + operationIsOffBy(_, pai, _, _, _, sink, _) and + state = TPointerArith(pai) + ) + } - /** - * Holds if `source` is an allocation that flows into the left-hand side of `pai`, which produces an out-of-bounds - * pointer that flows into an address that is dereferenced by `sink` (which is either a `LoadInstruction` or a - * `StoreInstruction`). The end result is that `sink` writes to an address that is off-by-`delta` from the end of - * the allocation. The string `operation` describes whether the `sink` is a load or a store (which is then used - * to produce the alert message). - * - * Note that multiple `delta`s can exist for a given `(source, pai, sink)` triplet. - */ - predicate hasFlowPath( - FinalFlow::PathNode source, FinalFlow::PathNode sink, PointerArithmeticInstruction pai, - string operation, int delta + predicate isAdditionalFlowStep( + DataFlow::Node node1, FlowState state1, DataFlow::Node node2, FlowState state2 ) { - FinalFlow::flowPath(source, sink) and - operationIsOffBy(source.getNode(), pai, _, _, operation, sink.getNode(), delta) and - sink.getState() = FinalConfig::TPointerArith(pai) + // A step from the left-hand side of a pointer-arithmetic operation that has been + // identified as creating an out-of-bounds pointer to the result of the pointer-arithmetic + // operation. + exists(PointerArithmeticInstruction pai | + pointerAddInstructionHasBounds(_, pai, node1, _) and + operationIsOffBy(_, pai, node2, _, _, _, _) and + state1 = TInitial() and + state2 = TPointerArith(pai) + ) + or + // A step from an out-of-bounds address to the operation (which is either a `StoreInstruction` + // or a `LoadInstruction`) that dereferences the address. + // This step exists purely for aesthetic reasons: we want the alert to be placed at the operation + // that causes the dereference, and not at the address that flows into the operation. + state1 = state2 and + exists(PointerArithmeticInstruction pai | + state1 = TPointerArith(pai) and + operationIsOffBy(_, pai, _, node1, _, node2, _) + ) } +} - from - FinalFlow::PathNode source, FinalFlow::PathNode sink, int k, string kstr, - PointerArithmeticInstruction pai, string operation, Expr offset, DataFlow::Node n - where - k = min(int cand | hasFlowPath(source, sink, pai, operation, cand)) and - offset = pai.getRight().getUnconvertedResultExpression() and - n = source.getNode() and - if k = 0 then kstr = "" else kstr = " + " + k - select sink.getNode(), source, sink, - "This " + operation + " might be out of bounds, as the pointer might be equal to $@ + $@" + kstr + - ".", n, n.toString(), offset, offset.toString() +module FinalFlow = DataFlow::GlobalWithState; + +/** + * Holds if `source` is an allocation that flows into the left-hand side of `pai`, which produces an out-of-bounds + * pointer that flows into an address that is dereferenced by `sink` (which is either a `LoadInstruction` or a + * `StoreInstruction`). The end result is that `sink` writes to an address that is off-by-`delta` from the end of + * the allocation. The string `operation` describes whether the `sink` is a load or a store (which is then used + * to produce the alert message). + * + * Note that multiple `delta`s can exist for a given `(source, pai, sink)` triplet. + */ +predicate hasFlowPath( + FinalFlow::PathNode source, FinalFlow::PathNode sink, PointerArithmeticInstruction pai, + string operation, int delta +) { + FinalFlow::flowPath(source, sink) and + operationIsOffBy(source.getNode(), pai, _, _, operation, sink.getNode(), delta) and + sink.getState() = FinalConfig::TPointerArith(pai) +} + +from + FinalFlow::PathNode source, FinalFlow::PathNode sink, int k, string kstr, + PointerArithmeticInstruction pai, string operation, Expr offset, DataFlow::Node n +where + k = min(int cand | hasFlowPath(source, sink, pai, operation, cand)) and + offset = pai.getRight().getUnconvertedResultExpression() and + n = source.getNode() and + if k = 0 then kstr = "" else kstr = " + " + k +select sink.getNode(), source, sink, + "This " + operation + " might be out of bounds, as the pointer might be equal to $@ + $@" + kstr + + ".", n, n.toString(), offset, offset.toString() From 6fb1058e73556970adcef2954f5aa0a054a7d7ce Mon Sep 17 00:00:00 2001 From: Geoffrey White <40627776+geoffw0@users.noreply.github.com> Date: Tue, 15 Aug 2023 10:46:36 +0100 Subject: [PATCH 036/122] Swift: Copy IncompleteHostnameRegex query from JS. --- .../IncompleteHostnameRegexShared.qll | 7 ++ .../swift/security/regex/HostnameRegex.qll | 18 +++++ .../CWE-020/IncompleteHostnameRegex.js | 9 +++ .../CWE-020/IncompleteHostnameRegex.qhelp | 73 +++++++++++++++++++ .../CWE-020/IncompleteHostnameRegex.ql | 16 ++++ .../CWE-020/IncompleteHostnameRegex.expected | 27 +++++++ .../CWE-020/IncompleteHostnameRegex.qlref | 1 + .../CWE-020/tst-IncompleteHostnameRegExp.js | 60 +++++++++++++++ 8 files changed, 211 insertions(+) create mode 100644 swift/ql/lib/codeql/swift/security/IncompleteHostnameRegexShared.qll create mode 100644 swift/ql/lib/codeql/swift/security/regex/HostnameRegex.qll create mode 100644 swift/ql/src/queries/Security/CWE-020/IncompleteHostnameRegex.js create mode 100644 swift/ql/src/queries/Security/CWE-020/IncompleteHostnameRegex.qhelp create mode 100644 swift/ql/src/queries/Security/CWE-020/IncompleteHostnameRegex.ql create mode 100644 swift/ql/test/query-tests/Security/CWE-020/IncompleteHostnameRegex.expected create mode 100644 swift/ql/test/query-tests/Security/CWE-020/IncompleteHostnameRegex.qlref create mode 100644 swift/ql/test/query-tests/Security/CWE-020/tst-IncompleteHostnameRegExp.js diff --git a/swift/ql/lib/codeql/swift/security/IncompleteHostnameRegexShared.qll b/swift/ql/lib/codeql/swift/security/IncompleteHostnameRegexShared.qll new file mode 100644 index 00000000000..524be45c653 --- /dev/null +++ b/swift/ql/lib/codeql/swift/security/IncompleteHostnameRegexShared.qll @@ -0,0 +1,7 @@ +/** + * Provides predicates for reasoning about regular expressions + * that match URLs and hostname patterns. + */ + +deprecated import semmle.javascript.security.regexp.HostnameRegexp as Dep +import Dep diff --git a/swift/ql/lib/codeql/swift/security/regex/HostnameRegex.qll b/swift/ql/lib/codeql/swift/security/regex/HostnameRegex.qll new file mode 100644 index 00000000000..c9853ce35f1 --- /dev/null +++ b/swift/ql/lib/codeql/swift/security/regex/HostnameRegex.qll @@ -0,0 +1,18 @@ +/** + * Provides predicates for reasoning about regular expressions + * that match URLs and hostname patterns. + */ + +private import javascript as JS +private import semmle.javascript.security.regexp.RegExpTreeView::RegExpTreeView as TreeImpl +private import semmle.javascript.Regexp as RegExp +private import codeql.regex.HostnameRegexp as Shared + +/** An implementation of the signature that allows the Hostname analysis to run. */ +module Impl implements Shared::HostnameRegexpSig { + class DataFlowNode = JS::DataFlow::Node; + + class RegExpPatternSource = RegExp::RegExpPatternSource; +} + +import Shared::Make diff --git a/swift/ql/src/queries/Security/CWE-020/IncompleteHostnameRegex.js b/swift/ql/src/queries/Security/CWE-020/IncompleteHostnameRegex.js new file mode 100644 index 00000000000..11d11a97ca4 --- /dev/null +++ b/swift/ql/src/queries/Security/CWE-020/IncompleteHostnameRegex.js @@ -0,0 +1,9 @@ +app.get('/some/path', function(req, res) { + let url = req.param('url'), + host = urlLib.parse(url).host; + // BAD: the host of `url` may be controlled by an attacker + let regex = /^((www|beta).)?example.com/; + if (host.match(regex)) { + res.redirect(url); + } +}); diff --git a/swift/ql/src/queries/Security/CWE-020/IncompleteHostnameRegex.qhelp b/swift/ql/src/queries/Security/CWE-020/IncompleteHostnameRegex.qhelp new file mode 100644 index 00000000000..3b49628c84f --- /dev/null +++ b/swift/ql/src/queries/Security/CWE-020/IncompleteHostnameRegex.qhelp @@ -0,0 +1,73 @@ + + + + +

    + + Sanitizing untrusted URLs is an important technique for + preventing attacks such as request forgeries and malicious + redirections. Often, this is done by checking that the host of a URL + is in a set of allowed hosts. + +

    + +

    + + If a regular expression implements such a check, it is + easy to accidentally make the check too permissive by not escaping the + . meta-characters appropriately. + + Even if the check is not used in a security-critical + context, the incomplete check may still cause undesirable behaviors + when it accidentally succeeds. + +

    +     + + +

    + + Escape all meta-characters appropriately when constructing + regular expressions for security checks, and pay special attention to the + . meta-character. + +

    +     + + + +

    + + The following example code checks that a URL redirection + will reach the example.com domain, or one of its + subdomains. + +

    + + + +

    + + The check is however easy to bypass because the unescaped + . allows for any character before + example.com, effectively allowing the redirect to go to + an attacker-controlled domain such as wwwXexample.com. + +

    +

    + + Address this vulnerability by escaping . + appropriately: let regex = /^((www|beta)\.)?example\.com/. + +

    + +     + + +
  • MDN: Regular Expressions
    • +
  • OWASP: SSRF
    • +
  • OWASP: XSS Unvalidated Redirects and Forwards Cheat Sheet.
    • +     +     diff --git a/swift/ql/src/queries/Security/CWE-020/IncompleteHostnameRegex.ql b/swift/ql/src/queries/Security/CWE-020/IncompleteHostnameRegex.ql new file mode 100644 index 00000000000..2bf62b710a0 --- /dev/null +++ b/swift/ql/src/queries/Security/CWE-020/IncompleteHostnameRegex.ql @@ -0,0 +1,16 @@ +/** + * @name Incomplete regular expression for hostnames + * @description Matching a URL or hostname against a regular expression that contains an unescaped dot as part of the hostname might match more hostnames than expected. + * @kind problem + * @problem.severity warning + * @security-severity 7.8 + * @precision high + * @id js/incomplete-hostname-regexp + * @tags correctness + * security + * external/cwe/cwe-020 + */ + +private import semmle.javascript.security.regexp.HostnameRegexp as HostnameRegexp + +query predicate problems = HostnameRegexp::incompleteHostnameRegExp/4; diff --git a/swift/ql/test/query-tests/Security/CWE-020/IncompleteHostnameRegex.expected b/swift/ql/test/query-tests/Security/CWE-020/IncompleteHostnameRegex.expected new file mode 100644 index 00000000000..d0fb91322fe --- /dev/null +++ b/swift/ql/test/query-tests/Security/CWE-020/IncompleteHostnameRegex.expected @@ -0,0 +1,27 @@ +| tst-IncompleteHostnameRegExp.js:3:3:3:28 | ^http:\\/\\/test.example.com | This regular expression has an unescaped '.' before 'example.com', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.js:3:2:3:29 | /^http: ... le.com/ | here | +| tst-IncompleteHostnameRegExp.js:5:3:5:28 | ^http:\\/\\/test.example.net | This regular expression has an unescaped '.' before 'example.net', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.js:5:2:5:29 | /^http: ... le.net/ | here | +| tst-IncompleteHostnameRegExp.js:6:3:6:42 | ^http:\\/\\/test.(example-a\|example-b).com | This regular expression has an unescaped '.' before '(example-a\|example-b).com', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.js:6:2:6:43 | /^http: ... b).com/ | here | +| tst-IncompleteHostnameRegExp.js:7:3:7:30 | ^http:\\/\\/(.+).example.com\\/ | This regular expression has an unescaped '.' before 'example.com', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.js:7:2:7:31 | /^http: ... .com\\// | here | +| tst-IncompleteHostnameRegExp.js:7:3:7:30 | ^http:\\/\\/(.+).example.com\\/ | This regular expression has an unrestricted wildcard '.+' which may cause 'example.com' to be matched anywhere in the URL, outside the hostname. | tst-IncompleteHostnameRegExp.js:7:2:7:31 | /^http: ... .com\\// | here | +| tst-IncompleteHostnameRegExp.js:10:3:10:36 | ^http:\\/\\/test.example.com\\/(?:.*) | This regular expression has an unescaped '.' before 'example.com', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.js:10:2:10:37 | /^http: ... (?:.*)/ | here | +| tst-IncompleteHostnameRegExp.js:11:14:11:37 | ^http://test.example.com | This regular expression has an unescaped '.' before 'example.com', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.js:11:13:11:38 | "^http: ... le.com" | here | +| tst-IncompleteHostnameRegExp.js:12:15:12:38 | ^http://test.example.com | This regular expression has an unescaped '.' before 'example.com', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.js:12:14:12:39 | "^http: ... le.com" | here | +| tst-IncompleteHostnameRegExp.js:15:23:15:46 | ^http://test.example.com | This string, which is used as a regular expression $@, has an unescaped '.' before 'example.com', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.js:15:13:15:50 | id(id(i ... com"))) | here | +| tst-IncompleteHostnameRegExp.js:19:18:19:34 | ^test.example.com | This string, which is used as a regular expression $@, has an unescaped '.' before 'example.com', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.js:20:13:20:26 | `${hostname}$` | here | +| tst-IncompleteHostnameRegExp.js:22:28:22:44 | test.example.com$ | This string, which is used as a regular expression $@, has an unescaped '.' before 'example.com', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.js:23:13:23:27 | domain.hostname | here | +| tst-IncompleteHostnameRegExp.js:28:24:28:40 | test.example.com$ | This string, which is used as a regular expression $@, has an unescaped '.' before 'example.com', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.js:26:21:26:35 | domain.hostname | here | +| tst-IncompleteHostnameRegExp.js:30:31:30:47 | test.example.com$ | This string, which is used as a regular expression $@, has an unescaped '.' before 'example.com', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.js:32:21:32:35 | domain.hostname | here | +| tst-IncompleteHostnameRegExp.js:37:3:37:53 | ^(https?:)?\\/\\/((service\|www).)?example.com(?=$\|\\/) | This regular expression has an unescaped '.' before 'example.com', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.js:37:2:37:54 | /^(http ... =$\|\\/)/ | here | +| tst-IncompleteHostnameRegExp.js:38:3:38:43 | ^(http\|https):\\/\\/www.example.com\\/p\\/f\\/ | This regular expression has an unescaped '.' before 'example.com', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.js:38:2:38:44 | /^(http ... p\\/f\\// | here | +| tst-IncompleteHostnameRegExp.js:39:5:39:30 | http:\\/\\/sub.example.com\\/ | This regular expression has an unescaped '.' before 'example.com', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.js:39:2:39:33 | /^(http ... om\\/)/g | here | +| tst-IncompleteHostnameRegExp.js:40:3:40:29 | ^https?:\\/\\/api.example.com | This regular expression has an unescaped '.' before 'example.com', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.js:40:2:40:30 | /^https ... le.com/ | here | +| tst-IncompleteHostnameRegExp.js:41:42:41:48 | ^https?://.+\\.example\\.com/ | This regular expression has an unrestricted wildcard '.+' which may cause 'example\\.com/' to be matched anywhere in the URL, outside the hostname. | tst-IncompleteHostnameRegExp.js:41:13:41:71 | '^http: ... \\.com/' | here | +| tst-IncompleteHostnameRegExp.js:43:3:43:32 | ^https:\\/\\/[a-z]*.example.com$ | This regular expression has an unescaped '.' before 'example.com', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.js:43:2:43:33 | /^https ... e.com$/ | here | +| tst-IncompleteHostnameRegExp.js:44:32:44:45 | .+.example.net | This regular expression has an unescaped '.' before 'example.net', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.js:44:9:44:101 | '^proto ... ernal)' | here | +| tst-IncompleteHostnameRegExp.js:44:47:44:62 | .+.example-a.com | This regular expression has an unescaped '.' before 'example-a.com', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.js:44:9:44:101 | '^proto ... ernal)' | here | +| tst-IncompleteHostnameRegExp.js:44:64:44:79 | .+.example-b.com | This regular expression has an unescaped '.' before 'example-b.com', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.js:44:9:44:101 | '^proto ... ernal)' | here | +| tst-IncompleteHostnameRegExp.js:48:42:48:47 | ^https?://.+.example\\.com/ | This regular expression has an unescaped '.' before 'example\\.com/', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.js:48:13:48:69 | '^http: ... \\.com/' | here | +| tst-IncompleteHostnameRegExp.js:48:42:48:47 | ^https?://.+.example\\.com/ | This regular expression has an unrestricted wildcard '.+' which may cause 'example\\.com/' to be matched anywhere in the URL, outside the hostname. | tst-IncompleteHostnameRegExp.js:48:13:48:69 | '^http: ... \\.com/' | here | +| tst-IncompleteHostnameRegExp.js:53:14:53:35 | test.example.com$ | This regular expression has an unescaped '.' before 'example.com', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.js:53:13:53:36 | 'test.' ... e.com$' | here | +| tst-IncompleteHostnameRegExp.js:55:14:55:38 | ^http://test.example.com | This regular expression has an unescaped '.' before 'example.com', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.js:55:13:55:39 | '^http: ... le.com' | here | +| tst-IncompleteHostnameRegExp.js:59:5:59:20 | foo.example\\.com | This regular expression has an unescaped '.' before 'example\\.com', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.js:59:2:59:32 | /^(foo. ... ever)$/ | here | diff --git a/swift/ql/test/query-tests/Security/CWE-020/IncompleteHostnameRegex.qlref b/swift/ql/test/query-tests/Security/CWE-020/IncompleteHostnameRegex.qlref new file mode 100644 index 00000000000..e818d947252 --- /dev/null +++ b/swift/ql/test/query-tests/Security/CWE-020/IncompleteHostnameRegex.qlref @@ -0,0 +1 @@ +Security/CWE-020/IncompleteHostnameRegExp.ql \ No newline at end of file diff --git a/swift/ql/test/query-tests/Security/CWE-020/tst-IncompleteHostnameRegExp.js b/swift/ql/test/query-tests/Security/CWE-020/tst-IncompleteHostnameRegExp.js new file mode 100644 index 00000000000..ddc267ebdd7 --- /dev/null +++ b/swift/ql/test/query-tests/Security/CWE-020/tst-IncompleteHostnameRegExp.js @@ -0,0 +1,60 @@ +(function() { + /^http:\/\/example.com/; // OK + /^http:\/\/test.example.com/; // NOT OK + /^http:\/\/test\\.example.com/; // OK + /^http:\/\/test.example.net/; // NOT OK + /^http:\/\/test.(example-a|example-b).com/; // NOT OK + /^http:\/\/(.+).example.com\//; // NOT OK + /^http:\/\/(\\.+)\\.example.com/; // OK + /^http:\/\/(?:.+)\\.test\\.example.com\//; // NOT OK + /^http:\/\/test.example.com\/(?:.*)/; // OK + new RegExp("^http://test.example.com"); // NOT OK + if (s.match("^http://test.example.com")) {} // NOT OK + + function id(e) { return e; } + new RegExp(id(id(id("^http://test.example.com")))); // NOT OK + + new RegExp(`test.example.com$`); // NOT OK + + let hostname = '^test.example.com'; // NOT OK + new RegExp(`${hostname}$`); + + let domain = { hostname: 'test.example.com$' }; // NOT OK + new RegExp(domain.hostname); + + function convert1(domain) { + return new RegExp(domain.hostname); + } + convert1({ hostname: 'test.example.com$' }); // NOT OK + + let domains = [ { hostname: 'test.example.com$' } ]; // NOT OK + function convert2(domain) { + return new RegExp(domain.hostname); + } + domains.map(d => convert2(d)); + + /^(.+\.(?:example-a|example-b)\.com)\//; // NOT OK + /^(https?:)?\/\/((service|www).)?example.com(?=$|\/)/; // NOT OK + /^(http|https):\/\/www.example.com\/p\/f\//; // NOT OK + /^(http:\/\/sub.example.com\/)/g; // NOT OK + /^https?:\/\/api.example.com/; // NOT OK + new RegExp('^http://localhost:8000|' + '^https?://.+\\.example\\.com/'); // NOT OK + new RegExp('^http[s]?:\/\/?sub1\\.sub2\\.example\\.com\/f\/(.+)'); // NOT OK + /^https:\/\/[a-z]*.example.com$/; // NOT OK + RegExp('^protos?://(localhost|.+.example.net|.+.example-a.com|.+.example-b.com|.+.example.internal)'); // NOT OK + + /^(example.dev|example.com)/; // OK + + new RegExp('^http://localhost:8000|' + '^https?://.+.example\\.com/'); // NOT OK + + var primary = 'example.com$'; + new RegExp('test.' + primary); // NOT OK, but not detected + + new RegExp('test.' + 'example.com$'); // NOT OK + + new RegExp('^http://test\.example.com'); // NOT OK + + /^http:\/\/(..|...)\.example\.com\/index\.html/; // OK, wildcards are intentional + /^http:\/\/.\.example\.com\/index\.html/; // OK, the wildcard is intentional + /^(foo.example\.com|whatever)$/; // kinda OK - one disjunction doesn't even look like a hostname +}); From 6deaf4e5f889bea2d7bfbda2000dc400f77889c4 Mon Sep 17 00:00:00 2001 From: Geoffrey White <40627776+geoffw0@users.noreply.github.com> Date: Wed, 16 Aug 2023 15:47:03 +0100 Subject: [PATCH 037/122] Swift: Rework ParsedStringRegex and introduce the needed RegexPatternSource class. --- swift/ql/lib/codeql/swift/regex/Regex.qll | 51 +++++++++++++++++++---- 1 file changed, 42 insertions(+), 9 deletions(-) diff --git a/swift/ql/lib/codeql/swift/regex/Regex.qll b/swift/ql/lib/codeql/swift/regex/Regex.qll index 0ae533d6843..6b7e57e06c5 100644 --- a/swift/ql/lib/codeql/swift/regex/Regex.qll +++ b/swift/ql/lib/codeql/swift/regex/Regex.qll @@ -8,6 +8,37 @@ private import codeql.swift.dataflow.DataFlow private import internal.ParseRegex private import internal.RegexTracking +/** + * A data flow node whose value may flow to a position where it is interpreted + * as a part of a regular expression. For example the string literal + * `"(a|b).*"` in: + * ``` + * Regex("(a|b).*").firstMatch(in: myString) + * ``` + */ +abstract class RegexPatternSource extends DataFlow::Node { + /** + * Gets a node where the pattern of this node is parsed as a part of + * a regular expression. + */ + abstract DataFlow::Node getAParse(); + + /** + * Gets the root term of the regular expression parsed from this pattern. + */ + abstract RegExpTerm getRegExpTerm(); +} + +/** + * For each `RegexPatternSource` data flow node, the corresponding `Expr` is + * a `Regex`. This is a simple wrapper to make that happen. + */ +private class RegexFromRegexPatternSource extends RegExp { + RegexPatternSource node; + + RegexFromRegexPatternSource() { this = node.asExpr() } +} + /** * A string literal that is used as a regular expression. For example * the string literal `"(a|b).*"` in: @@ -15,16 +46,18 @@ private import internal.RegexTracking * Regex("(a|b).*").firstMatch(in: myString) * ``` */ -private class ParsedStringRegex extends RegExp, StringLiteralExpr { +private class ParsedStringRegex extends RegexPatternSource { + StringLiteralExpr expr; DataFlow::Node use; - ParsedStringRegex() { StringLiteralUseFlow::flow(DataFlow::exprNode(this), use) } + ParsedStringRegex() { + expr = this.asExpr() and + StringLiteralUseFlow::flow(this, use) + } - /** - * Gets a dataflow node where this string literal is used as a regular - * expression. - */ - DataFlow::Node getUse() { result = use } + override DataFlow::Node getAParse() { result = use } + + override RegExpTerm getRegExpTerm() { result.getRegExp() = this.asExpr() } } /** @@ -246,11 +279,11 @@ abstract class RegexEval extends CallExpr { */ RegExp getARegex() { // string literal used directly as a regex - result.(ParsedStringRegex).getUse().asExpr() = this.getRegexInput() + DataFlow::exprNode(result).(ParsedStringRegex).getAParse().asExpr() = this.getRegexInput() or // string literal -> regex object -> use exists(RegexCreation regexCreation | - result.(ParsedStringRegex).getUse() = regexCreation.getStringInput() and + DataFlow::exprNode(result).(ParsedStringRegex).getAParse() = regexCreation.getStringInput() and RegexUseFlow::flow(regexCreation, DataFlow::exprNode(this.getRegexInput())) ) } From efcadbda693f483e656233b12ac407063c855f68 Mon Sep 17 00:00:00 2001 From: Geoffrey White <40627776+geoffw0@users.noreply.github.com> Date: Wed, 16 Aug 2023 17:13:51 +0100 Subject: [PATCH 038/122] Swift: Get the IncompleteHostnameRegex query working for Swift. --- .../security/IncompleteHostnameRegexShared.qll | 7 ------- .../codeql/swift/security/regex/HostnameRegex.qll | 15 +++++++++------ .../Security/CWE-020/IncompleteHostnameRegex.ql | 6 +++--- 3 files changed, 12 insertions(+), 16 deletions(-) delete mode 100644 swift/ql/lib/codeql/swift/security/IncompleteHostnameRegexShared.qll diff --git a/swift/ql/lib/codeql/swift/security/IncompleteHostnameRegexShared.qll b/swift/ql/lib/codeql/swift/security/IncompleteHostnameRegexShared.qll deleted file mode 100644 index 524be45c653..00000000000 --- a/swift/ql/lib/codeql/swift/security/IncompleteHostnameRegexShared.qll +++ /dev/null @@ -1,7 +0,0 @@ -/** - * Provides predicates for reasoning about regular expressions - * that match URLs and hostname patterns. - */ - -deprecated import semmle.javascript.security.regexp.HostnameRegexp as Dep -import Dep diff --git a/swift/ql/lib/codeql/swift/security/regex/HostnameRegex.qll b/swift/ql/lib/codeql/swift/security/regex/HostnameRegex.qll index c9853ce35f1..deb3899638f 100644 --- a/swift/ql/lib/codeql/swift/security/regex/HostnameRegex.qll +++ b/swift/ql/lib/codeql/swift/security/regex/HostnameRegex.qll @@ -3,16 +3,19 @@ * that match URLs and hostname patterns. */ -private import javascript as JS -private import semmle.javascript.security.regexp.RegExpTreeView::RegExpTreeView as TreeImpl -private import semmle.javascript.Regexp as RegExp +private import swift +private import codeql.swift.dataflow.DataFlow +private import codeql.swift.regex.Regex as Regex +private import codeql.swift.regex.RegexTreeView::RegexTreeView as TreeImpl private import codeql.regex.HostnameRegexp as Shared -/** An implementation of the signature that allows the Hostname analysis to run. */ +/** + * An implementation of the signature that allows the Hostname analysis to run. + */ module Impl implements Shared::HostnameRegexpSig { - class DataFlowNode = JS::DataFlow::Node; + class DataFlowNode = DataFlow::Node; - class RegExpPatternSource = RegExp::RegExpPatternSource; + class RegExpPatternSource = Regex::RegexPatternSource; } import Shared::Make diff --git a/swift/ql/src/queries/Security/CWE-020/IncompleteHostnameRegex.ql b/swift/ql/src/queries/Security/CWE-020/IncompleteHostnameRegex.ql index 2bf62b710a0..a9a49720e58 100644 --- a/swift/ql/src/queries/Security/CWE-020/IncompleteHostnameRegex.ql +++ b/swift/ql/src/queries/Security/CWE-020/IncompleteHostnameRegex.ql @@ -5,12 +5,12 @@ * @problem.severity warning * @security-severity 7.8 * @precision high - * @id js/incomplete-hostname-regexp + * @id swift/incomplete-hostname-regexp * @tags correctness * security * external/cwe/cwe-020 */ -private import semmle.javascript.security.regexp.HostnameRegexp as HostnameRegexp +private import codeql.swift.security.regex.HostnameRegex as HostnameRegex -query predicate problems = HostnameRegexp::incompleteHostnameRegExp/4; +query predicate problems = HostnameRegex::incompleteHostnameRegExp/4; From 1805b070dc58886f8ea8cbf3e0f6afe985b68209 Mon Sep 17 00:00:00 2001 From: Geoffrey White <40627776+geoffw0@users.noreply.github.com> Date: Wed, 16 Aug 2023 17:53:37 +0100 Subject: [PATCH 039/122] Swift: Adapt the IncompleteHostnameRegex test for Swift. --- .../CWE-020/IncompleteHostnameRegex.expected | 48 ++++---- .../CWE-020/IncompleteHostnameRegex.qlref | 2 +- .../query-tests/Security/CWE-020/test.swift | 113 ++++++++++++++++++ .../CWE-020/tst-IncompleteHostnameRegExp.js | 60 ---------- 4 files changed, 135 insertions(+), 88 deletions(-) create mode 100644 swift/ql/test/query-tests/Security/CWE-020/test.swift delete mode 100644 swift/ql/test/query-tests/Security/CWE-020/tst-IncompleteHostnameRegExp.js diff --git a/swift/ql/test/query-tests/Security/CWE-020/IncompleteHostnameRegex.expected b/swift/ql/test/query-tests/Security/CWE-020/IncompleteHostnameRegex.expected index d0fb91322fe..c0d5d15485c 100644 --- a/swift/ql/test/query-tests/Security/CWE-020/IncompleteHostnameRegex.expected +++ b/swift/ql/test/query-tests/Security/CWE-020/IncompleteHostnameRegex.expected @@ -1,27 +1,21 @@ -| tst-IncompleteHostnameRegExp.js:3:3:3:28 | ^http:\\/\\/test.example.com | This regular expression has an unescaped '.' before 'example.com', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.js:3:2:3:29 | /^http: ... le.com/ | here | -| tst-IncompleteHostnameRegExp.js:5:3:5:28 | ^http:\\/\\/test.example.net | This regular expression has an unescaped '.' before 'example.net', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.js:5:2:5:29 | /^http: ... le.net/ | here | -| tst-IncompleteHostnameRegExp.js:6:3:6:42 | ^http:\\/\\/test.(example-a\|example-b).com | This regular expression has an unescaped '.' before '(example-a\|example-b).com', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.js:6:2:6:43 | /^http: ... b).com/ | here | -| tst-IncompleteHostnameRegExp.js:7:3:7:30 | ^http:\\/\\/(.+).example.com\\/ | This regular expression has an unescaped '.' before 'example.com', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.js:7:2:7:31 | /^http: ... .com\\// | here | -| tst-IncompleteHostnameRegExp.js:7:3:7:30 | ^http:\\/\\/(.+).example.com\\/ | This regular expression has an unrestricted wildcard '.+' which may cause 'example.com' to be matched anywhere in the URL, outside the hostname. | tst-IncompleteHostnameRegExp.js:7:2:7:31 | /^http: ... .com\\// | here | -| tst-IncompleteHostnameRegExp.js:10:3:10:36 | ^http:\\/\\/test.example.com\\/(?:.*) | This regular expression has an unescaped '.' before 'example.com', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.js:10:2:10:37 | /^http: ... (?:.*)/ | here | -| tst-IncompleteHostnameRegExp.js:11:14:11:37 | ^http://test.example.com | This regular expression has an unescaped '.' before 'example.com', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.js:11:13:11:38 | "^http: ... le.com" | here | -| tst-IncompleteHostnameRegExp.js:12:15:12:38 | ^http://test.example.com | This regular expression has an unescaped '.' before 'example.com', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.js:12:14:12:39 | "^http: ... le.com" | here | -| tst-IncompleteHostnameRegExp.js:15:23:15:46 | ^http://test.example.com | This string, which is used as a regular expression $@, has an unescaped '.' before 'example.com', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.js:15:13:15:50 | id(id(i ... com"))) | here | -| tst-IncompleteHostnameRegExp.js:19:18:19:34 | ^test.example.com | This string, which is used as a regular expression $@, has an unescaped '.' before 'example.com', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.js:20:13:20:26 | `${hostname}$` | here | -| tst-IncompleteHostnameRegExp.js:22:28:22:44 | test.example.com$ | This string, which is used as a regular expression $@, has an unescaped '.' before 'example.com', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.js:23:13:23:27 | domain.hostname | here | -| tst-IncompleteHostnameRegExp.js:28:24:28:40 | test.example.com$ | This string, which is used as a regular expression $@, has an unescaped '.' before 'example.com', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.js:26:21:26:35 | domain.hostname | here | -| tst-IncompleteHostnameRegExp.js:30:31:30:47 | test.example.com$ | This string, which is used as a regular expression $@, has an unescaped '.' before 'example.com', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.js:32:21:32:35 | domain.hostname | here | -| tst-IncompleteHostnameRegExp.js:37:3:37:53 | ^(https?:)?\\/\\/((service\|www).)?example.com(?=$\|\\/) | This regular expression has an unescaped '.' before 'example.com', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.js:37:2:37:54 | /^(http ... =$\|\\/)/ | here | -| tst-IncompleteHostnameRegExp.js:38:3:38:43 | ^(http\|https):\\/\\/www.example.com\\/p\\/f\\/ | This regular expression has an unescaped '.' before 'example.com', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.js:38:2:38:44 | /^(http ... p\\/f\\// | here | -| tst-IncompleteHostnameRegExp.js:39:5:39:30 | http:\\/\\/sub.example.com\\/ | This regular expression has an unescaped '.' before 'example.com', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.js:39:2:39:33 | /^(http ... om\\/)/g | here | -| tst-IncompleteHostnameRegExp.js:40:3:40:29 | ^https?:\\/\\/api.example.com | This regular expression has an unescaped '.' before 'example.com', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.js:40:2:40:30 | /^https ... le.com/ | here | -| tst-IncompleteHostnameRegExp.js:41:42:41:48 | ^https?://.+\\.example\\.com/ | This regular expression has an unrestricted wildcard '.+' which may cause 'example\\.com/' to be matched anywhere in the URL, outside the hostname. | tst-IncompleteHostnameRegExp.js:41:13:41:71 | '^http: ... \\.com/' | here | -| tst-IncompleteHostnameRegExp.js:43:3:43:32 | ^https:\\/\\/[a-z]*.example.com$ | This regular expression has an unescaped '.' before 'example.com', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.js:43:2:43:33 | /^https ... e.com$/ | here | -| tst-IncompleteHostnameRegExp.js:44:32:44:45 | .+.example.net | This regular expression has an unescaped '.' before 'example.net', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.js:44:9:44:101 | '^proto ... ernal)' | here | -| tst-IncompleteHostnameRegExp.js:44:47:44:62 | .+.example-a.com | This regular expression has an unescaped '.' before 'example-a.com', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.js:44:9:44:101 | '^proto ... ernal)' | here | -| tst-IncompleteHostnameRegExp.js:44:64:44:79 | .+.example-b.com | This regular expression has an unescaped '.' before 'example-b.com', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.js:44:9:44:101 | '^proto ... ernal)' | here | -| tst-IncompleteHostnameRegExp.js:48:42:48:47 | ^https?://.+.example\\.com/ | This regular expression has an unescaped '.' before 'example\\.com/', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.js:48:13:48:69 | '^http: ... \\.com/' | here | -| tst-IncompleteHostnameRegExp.js:48:42:48:47 | ^https?://.+.example\\.com/ | This regular expression has an unrestricted wildcard '.+' which may cause 'example\\.com/' to be matched anywhere in the URL, outside the hostname. | tst-IncompleteHostnameRegExp.js:48:13:48:69 | '^http: ... \\.com/' | here | -| tst-IncompleteHostnameRegExp.js:53:14:53:35 | test.example.com$ | This regular expression has an unescaped '.' before 'example.com', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.js:53:13:53:36 | 'test.' ... e.com$' | here | -| tst-IncompleteHostnameRegExp.js:55:14:55:38 | ^http://test.example.com | This regular expression has an unescaped '.' before 'example.com', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.js:55:13:55:39 | '^http: ... le.com' | here | -| tst-IncompleteHostnameRegExp.js:59:5:59:20 | foo.example\\.com | This regular expression has an unescaped '.' before 'example\\.com', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.js:59:2:59:32 | /^(foo. ... ever)$/ | here | +| test.swift:60:17:60:40 | ^http://test.example.com/ | This regular expression has an unescaped '.' before 'example.com/', so it might match more hosts than expected. | test.swift:60:16:60:16 | ^http://test.example.com/ | here | +| test.swift:63:17:63:40 | ^http://test.example.net/ | This regular expression has an unescaped '.' before 'example.net/', so it might match more hosts than expected. | test.swift:63:16:63:16 | ^http://test.example.net/ | here | +| test.swift:64:17:64:54 | ^http://test.(example-a\|example-b).com/ | This regular expression has an unescaped '.' before '(example-a\|example-b).com/', so it might match more hosts than expected. | test.swift:64:16:64:16 | ^http://test.(example-a\|example-b).com/ | here | +| test.swift:65:17:65:40 | ^http://(.+).example.com/ | This regular expression has an unescaped '.' before 'example.com/', so it might match more hosts than expected. | test.swift:65:16:65:16 | ^http://(.+).example.com/ | here | +| test.swift:65:17:65:40 | ^http://(.+).example.com/ | This regular expression has an unrestricted wildcard '.+' which may cause 'example.com/' to be matched anywhere in the URL, outside the hostname. | test.swift:65:16:65:16 | ^http://(.+).example.com/ | here | +| test.swift:67:17:67:49 | ^http://(?:.+)\\.test\\.example.com/ | This regular expression has an unrestricted wildcard '.+' which may cause 'example.com/' to be matched anywhere in the URL, outside the hostname. | test.swift:67:16:67:16 | ^http://(?:.+)\\.test\\.example.com/ | here | +| test.swift:68:17:68:46 | ^http://test.example.com/(?:.*) | This regular expression has an unescaped '.' before 'example.com/', so it might match more hosts than expected. | test.swift:68:16:68:16 | ^http://test.example.com/(?:.*) | here | +| test.swift:70:17:70:63 | ^(https?:)?//((service\|www).)?example.com(?=$\|/) | This regular expression has an unescaped '.' before 'example.com', so it might match more hosts than expected. | test.swift:70:16:70:16 | ^(https?:)?//((service\|www).)?example.com(?=$\|/) | here | +| test.swift:71:17:71:51 | ^(http\|https)://www.example.com/p/f/ | This regular expression has an unescaped '.' before 'example.com/p/f/', so it might match more hosts than expected. | test.swift:71:16:71:16 | ^(http\|https)://www.example.com/p/f/ | here | +| test.swift:72:19:72:40 | http://sub.example.com/ | This regular expression has an unescaped '.' before 'example.com/', so it might match more hosts than expected. | test.swift:72:16:72:16 | ^(http://sub.example.com/) | here | +| test.swift:73:17:73:41 | ^https?://api.example.com/ | This regular expression has an unescaped '.' before 'example.com/', so it might match more hosts than expected. | test.swift:73:16:73:16 | ^https?://api.example.com/ | here | +| test.swift:75:17:75:43 | ^https://[a-z]*.example.com$ | This regular expression has an unescaped '.' before 'example.com', so it might match more hosts than expected. | test.swift:75:16:75:16 | ^https://[a-z]*.example.com$ | here | +| test.swift:77:39:77:51 | .+.example.net | This regular expression has an unescaped '.' before 'example.net', so it might match more hosts than expected. | test.swift:77:16:77:16 | ^protos?://(localhost\|.+.example.net\|.+.example-a.com\|.+.example-b.com\|.+.example.internal) | here | +| test.swift:77:54:77:68 | .+.example-a.com | This regular expression has an unescaped '.' before 'example-a.com', so it might match more hosts than expected. | test.swift:77:16:77:16 | ^protos?://(localhost\|.+.example.net\|.+.example-a.com\|.+.example-b.com\|.+.example.internal) | here | +| test.swift:77:71:77:85 | .+.example-b.com | This regular expression has an unescaped '.' before 'example-b.com', so it might match more hosts than expected. | test.swift:77:16:77:16 | ^protos?://(localhost\|.+.example.net\|.+.example-a.com\|.+.example-b.com\|.+.example.internal) | here | +| test.swift:81:19:81:33 | foo.example\\.com | This regular expression has an unescaped '.' before 'example\\.com', so it might match more hosts than expected. | test.swift:81:16:81:16 | ^(foo.example\\.com\|whatever)$ | here | +| test.swift:83:17:83:33 | ^test.example.com$ | This regular expression has an unescaped '.' before 'example.com', so it might match more hosts than expected. | test.swift:83:16:83:16 | ^test.example.com$ | here | +| test.swift:84:17:84:31 | test.example.com | This regular expression has an unescaped '.' before 'example.com', so it might match more hosts than expected. | test.swift:84:16:84:16 | test.example.com | here | +| test.swift:86:26:86:41 | test.example.com$ | This string, which is used as a regular expression $@, has an unescaped '.' before 'example.com', so it might match more hosts than expected. | test.swift:86:16:86:48 | call to id(_:) | here | +| test.swift:92:21:92:36 | test.example.com$ | This string, which is used as a regular expression $@, has an unescaped '.' before 'example.com', so it might match more hosts than expected. | test.swift:93:16:93:23 | .hostname | here | +| test.swift:98:29:98:44 | test.example.com$ | This string, which is used as a regular expression $@, has an unescaped '.' before 'example.com', so it might match more hosts than expected. | test.swift:96:20:96:27 | .hostname | here | diff --git a/swift/ql/test/query-tests/Security/CWE-020/IncompleteHostnameRegex.qlref b/swift/ql/test/query-tests/Security/CWE-020/IncompleteHostnameRegex.qlref index e818d947252..b80ac364258 100644 --- a/swift/ql/test/query-tests/Security/CWE-020/IncompleteHostnameRegex.qlref +++ b/swift/ql/test/query-tests/Security/CWE-020/IncompleteHostnameRegex.qlref @@ -1 +1 @@ -Security/CWE-020/IncompleteHostnameRegExp.ql \ No newline at end of file +queries/Security/CWE-020/IncompleteHostnameRegex.ql diff --git a/swift/ql/test/query-tests/Security/CWE-020/test.swift b/swift/ql/test/query-tests/Security/CWE-020/test.swift new file mode 100644 index 00000000000..07b3245c919 --- /dev/null +++ b/swift/ql/test/query-tests/Security/CWE-020/test.swift @@ -0,0 +1,113 @@ + +// --- stubs --- + +struct URL { + init?(string: String) {} +} + +extension String { + init(contentsOf: URL) { + let data = "" + self.init(data) + } +} + +struct AnyRegexOutput { +} + +protocol RegexComponent { + associatedtype RegexOutput +} + +struct Regex : RegexComponent { + struct Match { + } + + init(_ pattern: String) throws where Output == AnyRegexOutput { } + + func firstMatch(in string: String) throws -> Regex.Match? { return nil} + func wholeMatch(in string: String) throws -> Regex.Match? { return nil } + + typealias RegexOutput = Output +} + +extension String : RegexComponent { + typealias Output = Substring + typealias RegexOutput = String.Output +} + +// --- tests --- + +func id(_ s : String) -> String { return s } + +struct MyDomain { + init(_ hostname: String) { + self.hostname = hostname + } + + var hostname: String +} + +func testHostnames(myUrl: URL) throws { + let tainted = String(contentsOf: myUrl) // tainted + + _ = try Regex(#"^http://example\.com/"#).firstMatch(in: tainted) // GOOD + _ = try Regex(#"^http://example.com/"#).firstMatch(in: tainted) // GOOD (only '.' here gives a valid top-level domain) + _ = try Regex(#"^http://example.com"#).firstMatch(in: tainted) // BAD (missing anchor) [NOT DETECTED] + _ = try Regex(#"^http://test\.example\.com/"#).firstMatch(in: tainted) // GOOD + _ = try Regex(#"^http://test\.example.com/"#).firstMatch(in: tainted) // GOOD (only '.' here gives a valid top-level domain) + _ = try Regex(#"^http://test\.example.com"#).firstMatch(in: tainted) // BAD (missing anchor) [NOT DETECTED] + _ = try Regex(#"^http://test.example.com/"#).firstMatch(in: tainted) // BAD (incomplete hostname) + _ = try Regex(#"^http://test[.]example[.]com/"#).firstMatch(in: tainted) // GOOD (alternative method of escaping) + + _ = try Regex(#"^http://test.example.net/"#).firstMatch(in: tainted) // BAD (incomplete hostname) + _ = try Regex(#"^http://test.(example-a|example-b).com/"#).firstMatch(in: tainted) // BAD (incomplete hostname) + _ = try Regex(#"^http://(.+).example.com/"#).firstMatch(in: tainted) // BAD (incomplete hostname x 2) + _ = try Regex(#"^http://(\.+)\.example.com/"#).firstMatch(in: tainted) // GOOD + _ = try Regex(#"^http://(?:.+)\.test\.example.com/"#).firstMatch(in: tainted) // BAD (incomplete hostname) + _ = try Regex(#"^http://test.example.com/(?:.*)"#).firstMatch(in: tainted) // BAD (incomplete hostname) + _ = try Regex(#"^(.+\.(?:example-a|example-b)\.com)/"#).firstMatch(in: tainted) // BAD (missing anchor) [NOT DETECTED] + _ = try Regex(#"^(https?:)?//((service|www).)?example.com(?=$|/)"#).firstMatch(in: tainted) // BAD (incomplete hostname) + _ = try Regex(#"^(http|https)://www.example.com/p/f/"#).firstMatch(in: tainted) // BAD (incomplete hostname) + _ = try Regex(#"^(http://sub.example.com/)"#).firstMatch(in: tainted) // BAD (incomplete hostname) + _ = try Regex(#"^https?://api.example.com/"#).firstMatch(in: tainted) // BAD (incomplete hostname) + _ = try Regex(#"^http[s]?://?sub1\.sub2\.example\.com/f/(.+)"#).firstMatch(in: tainted) // GOOD (it has a capture group after the TLD, so should be ignored) + _ = try Regex(#"^https://[a-z]*.example.com$"#).firstMatch(in: tainted) // BAD (incomplete hostname) + _ = try Regex(#"^(example.dev|example.com)"#).firstMatch(in: tainted) // GOOD (any extended hostname wouldn't be included in the capture group) + _ = try Regex(#"^protos?://(localhost|.+.example.net|.+.example-a.com|.+.example-b.com|.+.example.internal)"#).firstMatch(in: tainted) // BAD (incomplete hostname x3, missing anchor x 1) + + _ = try Regex(#"^http://(..|...)\.example\.com/index\.html"#).firstMatch(in: tainted) // GOOD (wildcards are intentional) + _ = try Regex(#"^http://.\.example\.com/index\.html"#).firstMatch(in: tainted) // GOOD (the wildcard is intentional) + _ = try Regex(#"^(foo.example\.com|whatever)$"#).firstMatch(in: tainted) // DUBIOUS (one disjunction doesn't even look like a hostname) [DETECTED incomplete hostname] + + _ = try Regex(#"^test.example.com$"#).firstMatch(in: tainted) // BAD (incomplete hostname) + _ = try Regex(#"test.example.com"#).wholeMatch(in: tainted) // BAD (incomplete hostname, missing anchor) + + _ = try Regex(id(id(id(#"test.example.com$"#)))).firstMatch(in: tainted) // BAD (incomplete hostname) + + let hostname = #"test.example.com$"# // BAD (incomplete hostname) [NOT DETECTED] + _ = try Regex("\(hostname)").firstMatch(in: tainted) + + var domain = MyDomain("") + domain.hostname = #"test.example.com$"# // BAD (incomplete hostname) + _ = try Regex(domain.hostname).firstMatch(in: tainted) + + func convert1(_ domain: MyDomain) throws -> Regex { + return try Regex(domain.hostname) + } + _ = try convert1(MyDomain(#"test.example.com$"#)).firstMatch(in: tainted) // BAD (incomplete hostname) + + let domains = [ MyDomain(#"test.example.com$"#) ] // BAD (incomplete hostname) [NOT DETECTED] + func convert2(_ domain: MyDomain) throws -> Regex { + return try Regex(domain.hostname) + } + _ = try domains.map({ try convert2($0).firstMatch(in: tainted) }) + + let primary = "example.com$" + _ = try Regex("test." + primary).firstMatch(in: tainted) // BAD (incomplete hostname) [NOT DETECTED] + _ = try Regex("test." + "example.com$").firstMatch(in: tainted) // BAD (incomplete hostname) [NOT DETECTED] + _ = try Regex(#"^http://localhost:8000|" + "^https?://.+\.example\.com/"#).firstMatch(in: tainted) // BAD (incomplete hostname) [NOT DETECTED] + _ = try Regex(#"^http://localhost:8000|" + "^https?://.+.example\.com/"#).firstMatch(in: tainted) // BAD (incomplete hostname) [NOT DETECTED] + + let harmless = #"^http://test.example.com"# // GOOD (never used as a regex) +} diff --git a/swift/ql/test/query-tests/Security/CWE-020/tst-IncompleteHostnameRegExp.js b/swift/ql/test/query-tests/Security/CWE-020/tst-IncompleteHostnameRegExp.js deleted file mode 100644 index ddc267ebdd7..00000000000 --- a/swift/ql/test/query-tests/Security/CWE-020/tst-IncompleteHostnameRegExp.js +++ /dev/null @@ -1,60 +0,0 @@ -(function() { - /^http:\/\/example.com/; // OK - /^http:\/\/test.example.com/; // NOT OK - /^http:\/\/test\\.example.com/; // OK - /^http:\/\/test.example.net/; // NOT OK - /^http:\/\/test.(example-a|example-b).com/; // NOT OK - /^http:\/\/(.+).example.com\//; // NOT OK - /^http:\/\/(\\.+)\\.example.com/; // OK - /^http:\/\/(?:.+)\\.test\\.example.com\//; // NOT OK - /^http:\/\/test.example.com\/(?:.*)/; // OK - new RegExp("^http://test.example.com"); // NOT OK - if (s.match("^http://test.example.com")) {} // NOT OK - - function id(e) { return e; } - new RegExp(id(id(id("^http://test.example.com")))); // NOT OK - - new RegExp(`test.example.com$`); // NOT OK - - let hostname = '^test.example.com'; // NOT OK - new RegExp(`${hostname}$`); - - let domain = { hostname: 'test.example.com$' }; // NOT OK - new RegExp(domain.hostname); - - function convert1(domain) { - return new RegExp(domain.hostname); - } - convert1({ hostname: 'test.example.com$' }); // NOT OK - - let domains = [ { hostname: 'test.example.com$' } ]; // NOT OK - function convert2(domain) { - return new RegExp(domain.hostname); - } - domains.map(d => convert2(d)); - - /^(.+\.(?:example-a|example-b)\.com)\//; // NOT OK - /^(https?:)?\/\/((service|www).)?example.com(?=$|\/)/; // NOT OK - /^(http|https):\/\/www.example.com\/p\/f\//; // NOT OK - /^(http:\/\/sub.example.com\/)/g; // NOT OK - /^https?:\/\/api.example.com/; // NOT OK - new RegExp('^http://localhost:8000|' + '^https?://.+\\.example\\.com/'); // NOT OK - new RegExp('^http[s]?:\/\/?sub1\\.sub2\\.example\\.com\/f\/(.+)'); // NOT OK - /^https:\/\/[a-z]*.example.com$/; // NOT OK - RegExp('^protos?://(localhost|.+.example.net|.+.example-a.com|.+.example-b.com|.+.example.internal)'); // NOT OK - - /^(example.dev|example.com)/; // OK - - new RegExp('^http://localhost:8000|' + '^https?://.+.example\\.com/'); // NOT OK - - var primary = 'example.com$'; - new RegExp('test.' + primary); // NOT OK, but not detected - - new RegExp('test.' + 'example.com$'); // NOT OK - - new RegExp('^http://test\.example.com'); // NOT OK - - /^http:\/\/(..|...)\.example\.com\/index\.html/; // OK, wildcards are intentional - /^http:\/\/.\.example\.com\/index\.html/; // OK, the wildcard is intentional - /^(foo.example\.com|whatever)$/; // kinda OK - one disjunction doesn't even look like a hostname -}); From 03ca29ab96887a90ed4d85d9c298ef485f10bcac Mon Sep 17 00:00:00 2001 From: Geoffrey White <40627776+geoffw0@users.noreply.github.com> Date: Tue, 22 Aug 2023 18:33:28 +0100 Subject: [PATCH 040/122] Swift: Adapt the IncompleteHostnameRegex qhelp for Swift. --- .../Security/CWE-020/IncompleteHostnameRegex.js | 9 --------- .../Security/CWE-020/IncompleteHostnameRegex.qhelp | 11 ++++++----- .../CWE-020/IncompleteHostnameRegexBad.swift | 12 ++++++++++++ .../CWE-020/IncompleteHostnameRegexGood.swift | 12 ++++++++++++ 4 files changed, 30 insertions(+), 14 deletions(-) delete mode 100644 swift/ql/src/queries/Security/CWE-020/IncompleteHostnameRegex.js create mode 100644 swift/ql/src/queries/Security/CWE-020/IncompleteHostnameRegexBad.swift create mode 100644 swift/ql/src/queries/Security/CWE-020/IncompleteHostnameRegexGood.swift diff --git a/swift/ql/src/queries/Security/CWE-020/IncompleteHostnameRegex.js b/swift/ql/src/queries/Security/CWE-020/IncompleteHostnameRegex.js deleted file mode 100644 index 11d11a97ca4..00000000000 --- a/swift/ql/src/queries/Security/CWE-020/IncompleteHostnameRegex.js +++ /dev/null @@ -1,9 +0,0 @@ -app.get('/some/path', function(req, res) { - let url = req.param('url'), - host = urlLib.parse(url).host; - // BAD: the host of `url` may be controlled by an attacker - let regex = /^((www|beta).)?example.com/; - if (host.match(regex)) { - res.redirect(url); - } -}); diff --git a/swift/ql/src/queries/Security/CWE-020/IncompleteHostnameRegex.qhelp b/swift/ql/src/queries/Security/CWE-020/IncompleteHostnameRegex.qhelp index 3b49628c84f..803e8ddbe08 100644 --- a/swift/ql/src/queries/Security/CWE-020/IncompleteHostnameRegex.qhelp +++ b/swift/ql/src/queries/Security/CWE-020/IncompleteHostnameRegex.qhelp @@ -46,7 +46,7 @@

    - +

    @@ -59,15 +59,16 @@

    Address this vulnerability by escaping . - appropriately: let regex = /^((www|beta)\.)?example\.com/. + to \.:

    + +     -
  • MDN: Regular Expressions
    • -
  • OWASP: SSRF
    • -
  • OWASP: XSS Unvalidated Redirects and Forwards Cheat Sheet.
    • +
  • OWASP: Server Side Request Forgery
    • +
  • OWASP: Unvalidated Redirects and Forwards Cheat Sheet
    •  diff --git a/swift/ql/src/queries/Security/CWE-020/IncompleteHostnameRegexBad.swift b/swift/ql/src/queries/Security/CWE-020/IncompleteHostnameRegexBad.swift new file mode 100644 index 00000000000..c43294fdb23 --- /dev/null +++ b/swift/ql/src/queries/Security/CWE-020/IncompleteHostnameRegexBad.swift @@ -0,0 +1,12 @@ + +func handleUrl(_ urlString: String) { + // get the 'url=' parameter from the URL + let components = URLComponents(string: urlString) + let redirectParam = components?.queryItems?.first(where: { $0.name == "url" }) + + // check we trust the host + let regex = #/^(www|beta).example.com//# + if let match = redirectParam?.value?.firstMatch(of: regex) { + // ... trust the URL ... + } +} diff --git a/swift/ql/src/queries/Security/CWE-020/IncompleteHostnameRegexGood.swift b/swift/ql/src/queries/Security/CWE-020/IncompleteHostnameRegexGood.swift new file mode 100644 index 00000000000..082206836dc --- /dev/null +++ b/swift/ql/src/queries/Security/CWE-020/IncompleteHostnameRegexGood.swift @@ -0,0 +1,12 @@ + +func handleUrl(_ urlString: String) { + // get the 'url=' parameter from the URL + let components = URLComponents(string: urlString) + let redirectParam = components?.queryItems?.first(where: { $0.name == "url" }) + + // check we trust the host + let regex = #/^(www|beta)\.example\.com//# + if let match = redirectParam?.value?.firstMatch(of: regex) { + // ... trust the URL ... + } +} From 02fbb47b64f136e1aae5637fe83f0a0ce72f2639 Mon Sep 17 00:00:00 2001 From: Geoffrey White <40627776+geoffw0@users.noreply.github.com> Date: Wed, 23 Aug 2023 10:38:09 +0100 Subject: [PATCH 041/122] Swift: Change note. --- .../src/change-notes/2023-08-23-incomplete-hostname-regex.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 swift/ql/src/change-notes/2023-08-23-incomplete-hostname-regex.md diff --git a/swift/ql/src/change-notes/2023-08-23-incomplete-hostname-regex.md b/swift/ql/src/change-notes/2023-08-23-incomplete-hostname-regex.md new file mode 100644 index 00000000000..98daf65253d --- /dev/null +++ b/swift/ql/src/change-notes/2023-08-23-incomplete-hostname-regex.md @@ -0,0 +1,5 @@ +--- +category: newQuery +--- + +* Added new query "Incomplete regular expression for hostnames" (`swift/incomplete-hostname-regexp`). This query finds regular expressions matching a URL or hostname that might match more hostnames than expected. From 15c49eeee9d4880a158d182896330eb14f53994d Mon Sep 17 00:00:00 2001 From: Geoffrey White <40627776+geoffw0@users.noreply.github.com> Date: Wed, 23 Aug 2023 13:58:05 +0100 Subject: [PATCH 042/122] Update swift/ql/lib/codeql/swift/regex/Regex.qll Co-authored-by: Mathias Vorreiter Pedersen --- swift/ql/lib/codeql/swift/regex/Regex.qll | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swift/ql/lib/codeql/swift/regex/Regex.qll b/swift/ql/lib/codeql/swift/regex/Regex.qll index 6b7e57e06c5..795c15196db 100644 --- a/swift/ql/lib/codeql/swift/regex/Regex.qll +++ b/swift/ql/lib/codeql/swift/regex/Regex.qll @@ -57,7 +57,7 @@ private class ParsedStringRegex extends RegexPatternSource { override DataFlow::Node getAParse() { result = use } - override RegExpTerm getRegExpTerm() { result.getRegExp() = this.asExpr() } + override RegExpTerm getRegExpTerm() { result.getRegExp() = expr } } /** From 46fc1fdaa1f4d5fa1d706b347929deab28a3eaad Mon Sep 17 00:00:00 2001 From: Geoffrey White <40627776+geoffw0@users.noreply.github.com> Date: Wed, 23 Aug 2023 14:04:56 +0100 Subject: [PATCH 043/122] Swift: Suggestions from review. --- swift/ql/lib/codeql/swift/regex/Regex.qll | 4 +--- swift/ql/lib/codeql/swift/security/regex/HostnameRegex.qll | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/swift/ql/lib/codeql/swift/regex/Regex.qll b/swift/ql/lib/codeql/swift/regex/Regex.qll index 795c15196db..138edc061d1 100644 --- a/swift/ql/lib/codeql/swift/regex/Regex.qll +++ b/swift/ql/lib/codeql/swift/regex/Regex.qll @@ -34,9 +34,7 @@ abstract class RegexPatternSource extends DataFlow::Node { * a `Regex`. This is a simple wrapper to make that happen. */ private class RegexFromRegexPatternSource extends RegExp { - RegexPatternSource node; - - RegexFromRegexPatternSource() { this = node.asExpr() } + RegexFromRegexPatternSource() { this = any(RegexPatternSource node).asExpr() } } /** diff --git a/swift/ql/lib/codeql/swift/security/regex/HostnameRegex.qll b/swift/ql/lib/codeql/swift/security/regex/HostnameRegex.qll index deb3899638f..71612f16a6c 100644 --- a/swift/ql/lib/codeql/swift/security/regex/HostnameRegex.qll +++ b/swift/ql/lib/codeql/swift/security/regex/HostnameRegex.qll @@ -12,7 +12,7 @@ private import codeql.regex.HostnameRegexp as Shared /** * An implementation of the signature that allows the Hostname analysis to run. */ -module Impl implements Shared::HostnameRegexpSig { +private module Impl implements Shared::HostnameRegexpSig { class DataFlowNode = DataFlow::Node; class RegExpPatternSource = Regex::RegexPatternSource; From 123e58767bd3ca18b7625ccede7494ac0ecf5c0a Mon Sep 17 00:00:00 2001 From: Mathias Vorreiter Pedersen Date: Wed, 23 Aug 2023 22:42:00 +0100 Subject: [PATCH 044/122] C++: Share RangeAnalysisUtil with 'cpp/overrun-write'. --- .../new}/RangeAnalysisUtil.qll | 4 ++-- .../AllocationToInvalidPointer.qll | 2 +- .../InvalidPointerToDereference.qll | 2 +- .../CWE/CWE-119/OverrunWriteProductFlow.ql | 20 +------------------ 4 files changed, 5 insertions(+), 23 deletions(-) rename cpp/ql/lib/semmle/code/cpp/{security/InvalidPointerDereference => rangeanalysis/new}/RangeAnalysisUtil.qll (92%) diff --git a/cpp/ql/lib/semmle/code/cpp/security/InvalidPointerDereference/RangeAnalysisUtil.qll b/cpp/ql/lib/semmle/code/cpp/rangeanalysis/new/RangeAnalysisUtil.qll similarity index 92% rename from cpp/ql/lib/semmle/code/cpp/security/InvalidPointerDereference/RangeAnalysisUtil.qll rename to cpp/ql/lib/semmle/code/cpp/rangeanalysis/new/RangeAnalysisUtil.qll index 0e86b68418a..4fa2ce85e50 100644 --- a/cpp/ql/lib/semmle/code/cpp/security/InvalidPointerDereference/RangeAnalysisUtil.qll +++ b/cpp/ql/lib/semmle/code/cpp/rangeanalysis/new/RangeAnalysisUtil.qll @@ -1,6 +1,6 @@ /** - * This file contains the range-analysis specific parts of the `cpp/invalid-pointer-deref` query - * that is used by both `AllocationToInvalidPointer.qll` and `InvalidPointerToDereference.qll`. + * This file contains the range-analysis specific parts of the `cpp/invalid-pointer-deref` + * and `cpp/overrun-write` query. */ private import cpp diff --git a/cpp/ql/lib/semmle/code/cpp/security/InvalidPointerDereference/AllocationToInvalidPointer.qll b/cpp/ql/lib/semmle/code/cpp/security/InvalidPointerDereference/AllocationToInvalidPointer.qll index a49f7322e6a..05ca73caef4 100644 --- a/cpp/ql/lib/semmle/code/cpp/security/InvalidPointerDereference/AllocationToInvalidPointer.qll +++ b/cpp/ql/lib/semmle/code/cpp/security/InvalidPointerDereference/AllocationToInvalidPointer.qll @@ -56,7 +56,7 @@ private import semmle.code.cpp.ir.dataflow.internal.ProductFlow private import semmle.code.cpp.ir.ValueNumbering private import semmle.code.cpp.controlflow.IRGuards private import codeql.util.Unit -private import RangeAnalysisUtil +private import semmle.code.cpp.rangeanalysis.new.RangeAnalysisUtil private VariableAccess getAVariableAccess(Expr e) { e.getAChild*() = result } diff --git a/cpp/ql/lib/semmle/code/cpp/security/InvalidPointerDereference/InvalidPointerToDereference.qll b/cpp/ql/lib/semmle/code/cpp/security/InvalidPointerDereference/InvalidPointerToDereference.qll index 4a2175809e6..be097115941 100644 --- a/cpp/ql/lib/semmle/code/cpp/security/InvalidPointerDereference/InvalidPointerToDereference.qll +++ b/cpp/ql/lib/semmle/code/cpp/security/InvalidPointerDereference/InvalidPointerToDereference.qll @@ -81,7 +81,7 @@ private import semmle.code.cpp.dataflow.new.DataFlow private import semmle.code.cpp.ir.ValueNumbering private import semmle.code.cpp.controlflow.IRGuards private import AllocationToInvalidPointer as AllocToInvalidPointer -private import RangeAnalysisUtil +private import semmle.code.cpp.rangeanalysis.new.RangeAnalysisUtil private module InvalidPointerToDerefBarrier { private module BarrierConfig implements DataFlow::ConfigSig { diff --git a/cpp/ql/src/Security/CWE/CWE-119/OverrunWriteProductFlow.ql b/cpp/ql/src/Security/CWE/CWE-119/OverrunWriteProductFlow.ql index 510b7e8b6c4..f87acaf96e3 100644 --- a/cpp/ql/src/Security/CWE/CWE-119/OverrunWriteProductFlow.ql +++ b/cpp/ql/src/Security/CWE/CWE-119/OverrunWriteProductFlow.ql @@ -20,28 +20,10 @@ import semmle.code.cpp.models.interfaces.Allocation import semmle.code.cpp.models.interfaces.ArrayFunction import semmle.code.cpp.rangeanalysis.new.internal.semantic.analysis.RangeAnalysis import semmle.code.cpp.rangeanalysis.new.internal.semantic.SemanticExprSpecific +import semmle.code.cpp.rangeanalysis.new.RangeAnalysisUtil import StringSizeFlow::PathGraph1 import codeql.util.Unit -pragma[nomagic] -Instruction getABoundIn(SemBound b, IRFunction func) { - getSemanticExpr(result) = b.getExpr(0) and - result.getEnclosingIRFunction() = func -} - -/** - * Holds if `i <= b + delta`. - */ -bindingset[i] -pragma[inline_late] -predicate bounded(Instruction i, Instruction b, int delta) { - exists(SemBound bound, IRFunction func | - semBounded(getSemanticExpr(i), bound, delta, true, _) and - b = getABoundIn(bound, func) and - i.getEnclosingIRFunction() = func - ) -} - VariableAccess getAVariableAccess(Expr e) { e.getAChild*() = result } /** From 4e6707fee5a8c67792acc996afaf694f75219978 Mon Sep 17 00:00:00 2001 From: Mathias Vorreiter Pedersen Date: Thu, 24 Aug 2023 10:29:04 +0100 Subject: [PATCH 045/122] Fix joins. Before: ``` Pipeline standard for boundedSsa#8#ffffffff@d4d576wg was evaluated in 651 iterations totaling 39789ms (delta sizes total: 235714). 3482 ~0% {8} r1 = JOIN unequalIntegralSsa#5#fffff#prev_delta WITH boundedSsa#8#ffffffff#prev ON FIRST 4 OUTPUT Lhs.0, Lhs.1, Lhs.2, Lhs.3, Lhs.4, Rhs.4, Rhs.5, Rhs.6 1747 ~0% {8} r2 = SELECT r1 ON In.5 = false 1747 ~3% {8} r3 = SCAN r2 OUTPUT In.0, In.1, In.2, (In.3 + i2f(1)), false, In.6, In.7, In.4 1735 ~4% {8} r4 = SELECT r1 ON In.5 = true 1735 ~6% {8} r5 = SCAN r4 OUTPUT In.0, In.1, In.2, (In.3 - i2f(1)), true, In.6, In.7, In.4 1843 ~97% {8} r6 = JOIN unequalIntegralSsa#5#fffff#prev WITH boundedSsa#8#ffffffff#prev_delta ON FIRST 4 OUTPUT Lhs.0, Lhs.1, Lhs.2, Lhs.3, Lhs.4, Rhs.4, Rhs.5, Rhs.6 907 ~99% {8} r7 = SELECT r6 ON In.5 = false 907 ~108% {8} r8 = SCAN r7 OUTPUT In.0, In.1, In.2, (In.3 + i2f(1)), false, In.6, In.7, In.4 2642 ~41% {8} r9 = r5 UNION r8 4389 ~22% {8} r10 = r3 UNION r9 936 ~99% {8} r11 = SELECT r6 ON In.5 = true 936 ~100% {8} r12 = SCAN r11 OUTPUT In.0, In.1, In.2, (In.3 - i2f(1)), true, In.6, In.7, In.4 512745 ~2% {6} r13 = SCAN bounded#7#fffffff#prev_delta OUTPUT In.0, In.3, In.1, In.2, In.4, In.5 229218 ~0% {9} r14 = JOIN r13 WITH boundFlowStepSsa#6#ffffff_240135#join_rhs ON FIRST 2 OUTPUT Rhs.5, Rhs.2, Rhs.3, Lhs.2, Rhs.4, Lhs.3, Lhs.1, Lhs.4, Lhs.5 229218 ~0% {8} r15 = SCAN r14 OUTPUT In.0, In.1, In.2, In.3, (In.4 + In.5), In.6, In.7, In.8 {8} r16 = r15 AND NOT num#TSemNoReason#f(Lhs.0) 5397 ~0% {8} r17 = SCAN r16 OUTPUT In.1, In.2, In.3, In.4, In.5, In.6, In.7, In.0 557965 ~0% {5} r18 = SCAN unequalIntegralSsa#5#fffff#prev_delta OUTPUT In.0, In.2, In.3, In.1, In.4 0 ~0% {8} r19 = JOIN r18 WITH boundedPhi#7#fffffff#prev ON FIRST 3 OUTPUT Lhs.0, Lhs.3, Lhs.1, Lhs.2, Lhs.4, Rhs.3, Rhs.4, Rhs.5 0 ~0% {8} r20 = SELECT r19 ON In.5 = false 0 ~0% {8} r21 = SCAN r20 OUTPUT In.0, In.1, In.2, (In.3 + i2f(1)), false, In.6, In.7, In.4 5397 ~0% {8} r22 = r17 UNION r21 6333 ~12% {8} r23 = r12 UNION r22 10722 ~16% {8} r24 = r10 UNION r23 0 ~0% {8} r25 = SELECT r19 ON In.5 = true 0 ~0% {8} r26 = SCAN r25 OUTPUT In.0, In.1, In.2, (In.3 - i2f(1)), true, In.6, In.7, In.4 362119282 ~1% {5} r27 = SCAN unequalIntegralSsa#5#fffff#prev OUTPUT In.0, In.2, In.3, In.1, In.4 41 ~8% {8} r28 = JOIN r27 WITH boundedPhi#7#fffffff#prev_delta ON FIRST 3 OUTPUT Lhs.0, Lhs.3, Lhs.1, Lhs.2, Lhs.4, Rhs.3, Rhs.4, Rhs.5 27 ~0% {8} r29 = SELECT r28 ON In.5 = false 27 ~4% {8} r30 = SCAN r29 OUTPUT In.0, In.1, In.2, (In.3 + i2f(1)), false, In.6, In.7, In.4 14 ~42% {8} r31 = SELECT r28 ON In.5 = true 14 ~42% {8} r32 = SCAN r31 OUTPUT In.0, In.1, In.2, (In.3 - i2f(1)), true, In.6, In.7, In.4 41 ~12% {8} r33 = r30 UNION r32 41 ~12% {8} r34 = r26 UNION r33 512745 ~0% {7} r35 = SCAN bounded#7#fffffff#prev_delta OUTPUT In.0, In.3, In.1, In.2, In.4, In.5, In.6 229272 ~0% {10} r36 = JOIN r35 WITH boundFlowStepSsa#6#ffffff_240135#join_rhs ON FIRST 2 OUTPUT Rhs.5, Lhs.2, Lhs.1, Lhs.4, Lhs.5, Lhs.6, Rhs.2, Rhs.3, Rhs.4, Lhs.3 229272 ~0% {9} r37 = SCAN r36 OUTPUT In.0, In.1, In.2, In.3, In.4, In.5, In.6, In.7, (In.8 + In.9) 223867 ~0% {8} r38 = JOIN r37 WITH num#TSemNoReason#f ON FIRST 1 OUTPUT Lhs.6, Lhs.7, Lhs.1, Lhs.8, Lhs.2, Lhs.3, Lhs.4, Lhs.5 3482 ~1% {8} r39 = JOIN unequalIntegralSsa#5#fffff#prev_delta WITH boundedSsa#8#ffffffff#prev ON FIRST 4 OUTPUT Lhs.0, Lhs.1, Lhs.2, Lhs.3, Rhs.4, Rhs.5, Rhs.6, Rhs.7 1747 ~0% {8} r40 = SELECT r39 ON In.4 = false 1747 ~0% {8} r41 = SCAN r40 OUTPUT In.7, In.0, In.1, In.2, (In.3 + i2f(1)), false, In.5, In.6 1735 ~0% {8} r42 = SELECT r39 ON In.4 = true 1735 ~0% {8} r43 = SCAN r42 OUTPUT In.7, In.0, In.1, In.2, (In.3 - i2f(1)), true, In.5, In.6 3482 ~0% {8} r44 = r41 UNION r43 557965 ~0% {4} r45 = SCAN unequalIntegralSsa#5#fffff#prev_delta OUTPUT In.0, In.2, In.3, In.1 0 ~0% {8} r46 = JOIN r45 WITH boundedPhi#7#fffffff#prev ON FIRST 3 OUTPUT Lhs.0, Lhs.3, Lhs.1, Lhs.2, Rhs.3, Rhs.4, Rhs.5, Rhs.6 0 ~0% {8} r47 = SELECT r46 ON In.4 = false 0 ~0% {8} r48 = SCAN r47 OUTPUT In.7, In.0, In.1, In.2, (In.3 + i2f(1)), false, In.5, In.6 0 ~0% {8} r49 = SELECT r46 ON In.4 = true 0 ~0% {8} r50 = SCAN r49 OUTPUT In.7, In.0, In.1, In.2, (In.3 - i2f(1)), true, In.5, In.6 0 ~0% {8} r51 = r48 UNION r50 3482 ~0% {8} r52 = r44 UNION r51 {8} r53 = r52 AND NOT num#TSemNoReason#f(Lhs.0) 20 ~0% {8} r54 = SCAN r53 OUTPUT In.1, In.2, In.3, In.4, In.5, In.6, In.7, In.0 1843 ~0% {8} r55 = JOIN unequalIntegralSsa#5#fffff#prev WITH boundedSsa#8#ffffffff#prev_delta ON FIRST 4 OUTPUT Lhs.0, Lhs.1, Lhs.2, Lhs.3, Rhs.4, Rhs.5, Rhs.6, Rhs.7 907 ~0% {8} r56 = SELECT r55 ON In.4 = false 907 ~1% {8} r57 = SCAN r56 OUTPUT In.7, In.0, In.1, In.2, (In.3 + i2f(1)), false, In.5, In.6 936 ~0% {8} r58 = SELECT r55 ON In.4 = true 936 ~1% {8} r59 = SCAN r58 OUTPUT In.7, In.0, In.1, In.2, (In.3 - i2f(1)), true, In.5, In.6 1843 ~3% {8} r60 = r57 UNION r59 362119282 ~0% {4} r61 = SCAN unequalIntegralSsa#5#fffff#prev OUTPUT In.0, In.2, In.3, In.1 41 ~0% {8} r62 = JOIN r61 WITH boundedPhi#7#fffffff#prev_delta ON FIRST 3 OUTPUT Lhs.0, Lhs.3, Lhs.1, Lhs.2, Rhs.3, Rhs.4, Rhs.5, Rhs.6 27 ~0% {8} r63 = SELECT r62 ON In.4 = false 27 ~0% {8} r64 = SCAN r63 OUTPUT In.7, In.0, In.1, In.2, (In.3 + i2f(1)), false, In.5, In.6 14 ~0% {8} r65 = SELECT r62 ON In.4 = true 14 ~0% {8} r66 = SCAN r65 OUTPUT In.7, In.0, In.1, In.2, (In.3 - i2f(1)), true, In.5, In.6 41 ~0% {8} r67 = r64 UNION r66 1884 ~3% {8} r68 = r60 UNION r67 {8} r69 = r68 AND NOT num#TSemNoReason#f(Lhs.0) 1853 ~0% {8} r70 = SCAN r69 OUTPUT In.1, In.2, In.3, In.4, In.5, In.6, In.7, In.0 1873 ~0% {8} r71 = r54 UNION r70 225740 ~0% {8} r72 = r38 UNION r71 225781 ~0% {8} r73 = r34 UNION r72 236503 ~2% {8} r74 = r24 UNION r73 235722 ~1% {8} r75 = r74 AND NOT boundedSsa#8#ffffffff#prev(Lhs.0, Lhs.1, Lhs.2, Lhs.3, Lhs.4, Lhs.5, Lhs.6, Lhs.7) return r75 ``` After: ``` Pipeline standard for boundedSsa#8#ffffffff@2122e6w8 was evaluated in 651 iterations totaling 343ms (delta sizes total: 235815). 0 ~0% {8} r1 = JOIN unequalIntegralSsa#5#fffff#prev_delta WITH boundedPhi#7#fffffff#prev ON FIRST 3 OUTPUT Lhs.0, Lhs.1, Lhs.2, Lhs.3, Lhs.4, Rhs.3, Rhs.4, Rhs.5 0 ~0% {8} r2 = SELECT r1 ON In.5 = false 0 ~0% {8} r3 = SCAN r2 OUTPUT In.0, In.1, (In.2 + i2f(1)), In.3, false, In.6, In.7, In.4 0 ~0% {8} r4 = SELECT r1 ON In.5 = true 0 ~0% {8} r5 = SCAN r4 OUTPUT In.0, In.1, (In.2 - i2f(1)), In.3, true, In.6, In.7, In.4 41 ~8% {8} r6 = JOIN unequalIntegralSsa#5#fffff#prev WITH boundedPhi#7#fffffff#prev_delta ON FIRST 3 OUTPUT Lhs.0, Lhs.1, Lhs.2, Lhs.3, Lhs.4, Rhs.3, Rhs.4, Rhs.5 27 ~0% {8} r7 = SELECT r6 ON In.5 = false 27 ~0% {8} r8 = SCAN r7 OUTPUT In.0, In.1, (In.2 + i2f(1)), In.3, false, In.6, In.7, In.4 27 ~0% {8} r9 = r5 UNION r8 27 ~0% {8} r10 = r3 UNION r9 14 ~42% {8} r11 = SELECT r6 ON In.5 = true 14 ~42% {8} r12 = SCAN r11 OUTPUT In.0, In.1, (In.2 - i2f(1)), In.3, true, In.6, In.7, In.4 3482 ~0% {8} r13 = JOIN unequalIntegralSsa#5#fffff#prev_delta WITH boundedSsa#8#ffffffff#prev ON FIRST 4 OUTPUT Lhs.0, Lhs.1, Lhs.2, Lhs.3, Lhs.4, Rhs.4, Rhs.5, Rhs.6 1747 ~0% {8} r14 = SELECT r13 ON In.5 = false 1747 ~3% {8} r15 = SCAN r14 OUTPUT In.0, In.1, (In.2 + i2f(1)), In.3, false, In.6, In.7, In.4 1735 ~3% {8} r16 = SELECT r13 ON In.5 = true 1735 ~3% {8} r17 = SCAN r16 OUTPUT In.0, In.1, (In.2 - i2f(1)), In.3, true, In.6, In.7, In.4 3482 ~0% {8} r18 = r15 UNION r17 3496 ~0% {8} r19 = r12 UNION r18 3523 ~0% {8} r20 = r10 UNION r19 1843 ~103% {8} r21 = JOIN unequalIntegralSsa#5#fffff#prev WITH boundedSsa#8#ffffffff#prev_delta ON FIRST 4 OUTPUT Lhs.0, Lhs.1, Lhs.2, Lhs.3, Lhs.4, Rhs.4, Rhs.5, Rhs.6 907 ~105% {8} r22 = SELECT r21 ON In.5 = false 907 ~108% {8} r23 = SCAN r22 OUTPUT In.0, In.1, (In.2 + i2f(1)), In.3, false, In.6, In.7, In.4 936 ~101% {8} r24 = SELECT r21 ON In.5 = true 936 ~109% {8} r25 = SCAN r24 OUTPUT In.0, In.1, (In.2 - i2f(1)), In.3, true, In.6, In.7, In.4 512745 ~4% {6} r26 = SCAN bounded#7#fffffff#prev_delta OUTPUT In.0, In.3, In.1, In.2, In.4, In.5 229313 ~0% {9} r27 = JOIN r26 WITH boundFlowStepSsa#6#ffffff_240135#join_rhs ON FIRST 2 OUTPUT Rhs.5, Rhs.2, Lhs.2, Rhs.4, Lhs.3, Rhs.3, Lhs.1, Lhs.4, Lhs.5 229313 ~0% {8} r28 = SCAN r27 OUTPUT In.0, In.1, In.2, (In.3 + In.4), In.5, In.6, In.7, In.8 {8} r29 = r28 AND NOT num#TSemNoReason#f(Lhs.0) 5492 ~2% {8} r30 = SCAN r29 OUTPUT In.1, In.2, In.3, In.4, In.5, In.6, In.7, In.0 6428 ~15% {8} r31 = r25 UNION r30 7335 ~27% {8} r32 = r23 UNION r31 512745 ~0% {7} r33 = SCAN bounded#7#fffffff#prev_delta OUTPUT In.0, In.3, In.1, In.2, In.4, In.5, In.6 229367 ~0% {10} r34 = JOIN r33 WITH boundFlowStepSsa#6#ffffff_240135#join_rhs ON FIRST 2 OUTPUT Rhs.5, Lhs.2, Lhs.1, Lhs.4, Lhs.5, Lhs.6, Rhs.2, Rhs.3, Rhs.4, Lhs.3 229367 ~0% {9} r35 = SCAN r34 OUTPUT In.0, In.1, In.2, In.3, In.4, In.5, In.6, In.7, (In.8 + In.9) 223867 ~0% {8} r36 = JOIN r35 WITH num#TSemNoReason#f ON FIRST 1 OUTPUT Lhs.6, Lhs.1, Lhs.8, Lhs.7, Lhs.2, Lhs.3, Lhs.4, Lhs.5 0 ~0% {8} r37 = JOIN unequalIntegralSsa#5#fffff#prev_delta WITH boundedPhi#7#fffffff#prev ON FIRST 3 OUTPUT Lhs.0, Lhs.1, Lhs.2, Lhs.3, Rhs.3, Rhs.4, Rhs.5, Rhs.6 0 ~0% {8} r38 = SELECT r37 ON In.4 = false 0 ~0% {8} r39 = SCAN r38 OUTPUT In.7, In.0, In.1, (In.2 + i2f(1)), In.3, false, In.5, In.6 0 ~0% {8} r40 = SELECT r37 ON In.4 = true 0 ~0% {8} r41 = SCAN r40 OUTPUT In.7, In.0, In.1, (In.2 - i2f(1)), In.3, true, In.5, In.6 0 ~0% {8} r42 = r39 UNION r41 3482 ~0% {8} r43 = JOIN unequalIntegralSsa#5#fffff#prev_delta WITH boundedSsa#8#ffffffff#prev ON FIRST 4 OUTPUT Lhs.0, Lhs.1, Lhs.2, Lhs.3, Rhs.4, Rhs.5, Rhs.6, Rhs.7 1747 ~0% {8} r44 = SELECT r43 ON In.4 = false 1747 ~0% {8} r45 = SCAN r44 OUTPUT In.7, In.0, In.1, (In.2 + i2f(1)), In.3, false, In.5, In.6 1735 ~1% {8} r46 = SELECT r43 ON In.4 = true 1735 ~2% {8} r47 = SCAN r46 OUTPUT In.7, In.0, In.1, (In.2 - i2f(1)), In.3, true, In.5, In.6 3482 ~2% {8} r48 = r45 UNION r47 3482 ~2% {8} r49 = r42 UNION r48 {8} r50 = r49 AND NOT num#TSemNoReason#f(Lhs.0) 20 ~0% {8} r51 = SCAN r50 OUTPUT In.1, In.2, In.3, In.4, In.5, In.6, In.7, In.0 41 ~2% {8} r52 = JOIN unequalIntegralSsa#5#fffff#prev WITH boundedPhi#7#fffffff#prev_delta ON FIRST 3 OUTPUT Lhs.0, Lhs.1, Lhs.2, Lhs.3, Rhs.3, Rhs.4, Rhs.5, Rhs.6 27 ~0% {8} r53 = SELECT r52 ON In.4 = false 27 ~0% {8} r54 = SCAN r53 OUTPUT In.7, In.0, In.1, (In.2 + i2f(1)), In.3, false, In.5, In.6 14 ~0% {8} r55 = SELECT r52 ON In.4 = true 14 ~0% {8} r56 = SCAN r55 OUTPUT In.7, In.0, In.1, (In.2 - i2f(1)), In.3, true, In.5, In.6 41 ~0% {8} r57 = r54 UNION r56 1843 ~1% {8} r58 = JOIN unequalIntegralSsa#5#fffff#prev WITH boundedSsa#8#ffffffff#prev_delta ON FIRST 4 OUTPUT Lhs.0, Lhs.1, Lhs.2, Lhs.3, Rhs.4, Rhs.5, Rhs.6, Rhs.7 907 ~0% {8} r59 = SELECT r58 ON In.4 = false 907 ~1% {8} r60 = SCAN r59 OUTPUT In.7, In.0, In.1, (In.2 + i2f(1)), In.3, false, In.5, In.6 936 ~0% {8} r61 = SELECT r58 ON In.4 = true 936 ~0% {8} r62 = SCAN r61 OUTPUT In.7, In.0, In.1, (In.2 - i2f(1)), In.3, true, In.5, In.6 1843 ~0% {8} r63 = r60 UNION r62 1884 ~0% {8} r64 = r57 UNION r63 {8} r65 = r64 AND NOT num#TSemNoReason#f(Lhs.0) 1853 ~1% {8} r66 = SCAN r65 OUTPUT In.1, In.2, In.3, In.4, In.5, In.6, In.7, In.0 1873 ~1% {8} r67 = r51 UNION r66 225740 ~0% {8} r68 = r36 UNION r67 233075 ~0% {8} r69 = r32 UNION r68 236598 ~0% {8} r70 = r20 UNION r69 235817 ~0% {8} r71 = r70 AND NOT boundedSsa#8#ffffffff#prev(Lhs.0, Lhs.1, Lhs.2, Lhs.3, Lhs.4, Lhs.5, Lhs.6, Lhs.7) return r71 ``` --- .../semantic/analysis/RangeAnalysisStage.qll | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/cpp/ql/lib/semmle/code/cpp/rangeanalysis/new/internal/semantic/analysis/RangeAnalysisStage.qll b/cpp/ql/lib/semmle/code/cpp/rangeanalysis/new/internal/semantic/analysis/RangeAnalysisStage.qll index 0bd665ed10c..8f701cbc111 100644 --- a/cpp/ql/lib/semmle/code/cpp/rangeanalysis/new/internal/semantic/analysis/RangeAnalysisStage.qll +++ b/cpp/ql/lib/semmle/code/cpp/rangeanalysis/new/internal/semantic/analysis/RangeAnalysisStage.qll @@ -660,7 +660,7 @@ module RangeStage< * - `upper = false` : `v >= b + delta` */ private predicate boundedSsa( - SemSsaVariable v, SemSsaReadPosition pos, SemBound b, D::Delta delta, boolean upper, + SemSsaVariable v, SemBound b, D::Delta delta, SemSsaReadPosition pos, boolean upper, boolean fromBackEdge, D::Delta origdelta, SemReason reason ) { exists(SemExpr mid, D::Delta d1, D::Delta d2, SemReason r1, SemReason r2 | @@ -673,10 +673,13 @@ module RangeStage< ) or exists(D::Delta d, SemReason r1, SemReason r2 | - boundedSsa(v, pos, b, d, upper, fromBackEdge, origdelta, r2) or - boundedPhi(v, b, d, upper, fromBackEdge, origdelta, r2) + boundedSsa(pragma[only_bind_into](v), pragma[only_bind_into](b), pragma[only_bind_into](d), + pragma[only_bind_into](pos), upper, fromBackEdge, origdelta, r2) + or + boundedPhi(pragma[only_bind_into](v), pragma[only_bind_into](b), pragma[only_bind_into](d), + upper, fromBackEdge, origdelta, r2) | - unequalIntegralSsa(v, pos, b, d, r1) and + unequalIntegralSsa(v, b, d, pos, r1) and ( upper = true and delta = D::fromFloat(D::toFloat(d) - 1) or @@ -694,7 +697,7 @@ module RangeStage< * Holds if `v != b + delta` at `pos` and `v` is of integral type. */ private predicate unequalIntegralSsa( - SemSsaVariable v, SemSsaReadPosition pos, SemBound b, D::Delta delta, SemReason reason + SemSsaVariable v, SemBound b, D::Delta delta, SemSsaReadPosition pos, SemReason reason ) { exists(SemExpr e, D::Delta d1, D::Delta d2 | unequalFlowStepIntegralSsa(v, pos, e, d1, reason) and @@ -746,7 +749,7 @@ module RangeStage< ) { edge.phiInput(phi, inp) and exists(D::Delta d, boolean fromBackEdge0 | - boundedSsa(inp, edge, b, d, upper, fromBackEdge0, origdelta, reason) + boundedSsa(inp, b, d, edge, upper, fromBackEdge0, origdelta, reason) or boundedPhi(inp, b, d, upper, fromBackEdge0, origdelta, reason) or @@ -1022,7 +1025,7 @@ module RangeStage< reason = TSemNoReason() or exists(SemSsaVariable v, SemSsaReadPositionBlock bb | - boundedSsa(v, bb, b, delta, upper, fromBackEdge, origdelta, reason) and + boundedSsa(v, b, delta, bb, upper, fromBackEdge, origdelta, reason) and e = v.getAUse() and bb.getBlock() = e.getBasicBlock() ) From d42e89209712fcb1a44b442fb72fef52b68a18c0 Mon Sep 17 00:00:00 2001 From: Mathias Vorreiter Pedersen Date: Thu, 24 Aug 2023 12:20:41 +0100 Subject: [PATCH 046/122] Fix more joins. Before: ``` Tuple counts for valueFlowStepSsa#4#ffff/4@2cddce6j after 11.4s: 11571217 ~3% {3} r1 = SCAN semSsaUpdateStep#3#fff OUTPUT In.0 'v', In.1 'e', f2i(In.2) 11992425 ~0% {4} r2 = JOIN r1 WITH SemanticSSA#aa9d1d08::SemSsaReadPosition::hasReadOfVar#1#dispred#ff_10#join_rhs ON FIRST 1 OUTPUT Lhs.0 'v', Rhs.1 'pos', Lhs.1 'e', Lhs.2 'delta' 869481225 ~2% {4} r3 = JOIN SemanticGuard#7b46a302::semGuardDirectlyControlsSsaRead#3#fff_102#join_rhs WITH SemanticSSA#aa9d1d08::SemSsaReadPosition::hasReadOfVar#1#dispred#ff ON FIRST 1 OUTPUT Rhs.1 'v', Lhs.2, Lhs.1, Lhs.0 'pos' 5749 ~0% {4} r4 = JOIN r3 WITH semEqFlowCond#5#fffbff#cpe#12356_03412#join_rhs ON FIRST 3 OUTPUT Lhs.3 'pos', Lhs.0 'v', Rhs.3 'e', Rhs.4 5749 ~1% {4} r5 = SCAN r4 OUTPUT In.0 'pos', In.1 'v', In.2 'e', f2i(In.3) 5749 ~0% {4} r6 = SCAN r5 OUTPUT In.1 'v', In.0 'pos', In.2 'e', In.3 'delta' 11998174 ~0% {4} r7 = r2 UNION r6 return r7 ``` After: ``` Tuple counts for valueFlowStepSsaEqFlowCond#4#ffff/4@f196e4ok after 37ms: 59567 ~0% {5} r1 = JOIN const_true WITH semEqFlowCond#5#ffffff_301245#join_rhs ON FIRST 1 OUTPUT Rhs.1 'v', Rhs.2 'e', Rhs.4, Rhs.5, Rhs.3 59567 ~0% {5} r2 = SCAN r1 OUTPUT In.0 'v', In.1 'e', In.2, In.3, f2i(In.4) 59567 ~4% {5} r3 = SCAN r2 OUTPUT In.3, In.2, In.0 'v', In.1 'e', In.4 'delta' 176881 ~0% {4} r4 = JOIN r3 WITH SemanticGuard#7b46a302::semGuardDirectlyControlsSsaRead#3#fff_021#join_rhs ON FIRST 2 OUTPUT Rhs.2 'pos', Lhs.2 'v', Lhs.3 'e', Lhs.4 'delta' return r4 Tuple counts for valueFlowStepSsa#4#ffff/4@e22d39v5 after 1s: 5749 ~0% {4} r1 = JOIN SemanticSSA#aa9d1d08::SemSsaReadPosition::hasReadOfVar#1#dispred#ff WITH valueFlowStepSsaEqFlowCond#4#ffff ON FIRST 2 OUTPUT Lhs.1 'v', Lhs.0 'pos', Rhs.2 'e', Rhs.3 'delta' 11571217 ~0% {3} r2 = SCAN semSsaUpdateStep#3#fff OUTPUT In.0 'v', In.1 'e', f2i(In.2) 11992425 ~0% {4} r3 = JOIN r2 WITH SemanticSSA#aa9d1d08::SemSsaReadPosition::hasReadOfVar#1#dispred#ff_10#join_rhs ON FIRST 1 OUTPUT Lhs.0 'v', Rhs.1 'pos', Lhs.1 'e', Lhs.2 'delta' 11998174 ~0% {4} r4 = r1 UNION r3 return r4 ``` --- .../new/internal/semantic/SemanticGuard.qll | 1 + .../semantic/analysis/ModulusAnalysis.qll | 22 +++++++++++++------ .../internal/semantic/analysis/RangeUtils.qll | 1 + 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/cpp/ql/lib/semmle/code/cpp/rangeanalysis/new/internal/semantic/SemanticGuard.qll b/cpp/ql/lib/semmle/code/cpp/rangeanalysis/new/internal/semantic/SemanticGuard.qll index 8faf6a3a1de..e7f77c31486 100644 --- a/cpp/ql/lib/semmle/code/cpp/rangeanalysis/new/internal/semantic/SemanticGuard.qll +++ b/cpp/ql/lib/semmle/code/cpp/rangeanalysis/new/internal/semantic/SemanticGuard.qll @@ -39,6 +39,7 @@ predicate semImplies_v2(SemGuard g1, boolean b1, SemGuard g2, boolean b2) { * Holds if `guard` directly controls the position `controlled` with the * value `testIsTrue`. */ +pragma[nomagic] predicate semGuardDirectlyControlsSsaRead( SemGuard guard, SemSsaReadPosition controlled, boolean testIsTrue ) { diff --git a/cpp/ql/lib/semmle/code/cpp/rangeanalysis/new/internal/semantic/analysis/ModulusAnalysis.qll b/cpp/ql/lib/semmle/code/cpp/rangeanalysis/new/internal/semantic/analysis/ModulusAnalysis.qll index 42632f602de..9b00aca362f 100644 --- a/cpp/ql/lib/semmle/code/cpp/rangeanalysis/new/internal/semantic/analysis/ModulusAnalysis.qll +++ b/cpp/ql/lib/semmle/code/cpp/rangeanalysis/new/internal/semantic/analysis/ModulusAnalysis.qll @@ -17,19 +17,27 @@ private import RangeUtils private import RangeAnalysisStage module ModulusAnalysis Bounds, UtilSig U> { - /** - * Holds if `e + delta` equals `v` at `pos`. - */ - private predicate valueFlowStepSsa(SemSsaVariable v, SemSsaReadPosition pos, SemExpr e, int delta) { - U::semSsaUpdateStep(v, e, D::fromInt(delta)) and pos.hasReadOfVar(v) - or + pragma[nomagic] + private predicate valueFlowStepSsaEqFlowCond( + SemSsaReadPosition pos, SemSsaVariable v, SemExpr e, int delta + ) { exists(SemGuard guard, boolean testIsTrue | - pos.hasReadOfVar(v) and guard = U::semEqFlowCond(v, e, D::fromInt(delta), true, testIsTrue) and semGuardDirectlyControlsSsaRead(guard, pos, testIsTrue) ) } + /** + * Holds if `e + delta` equals `v` at `pos`. + */ + pragma[nomagic] + private predicate valueFlowStepSsa(SemSsaVariable v, SemSsaReadPosition pos, SemExpr e, int delta) { + U::semSsaUpdateStep(v, e, D::fromInt(delta)) and pos.hasReadOfVar(v) + or + pos.hasReadOfVar(v) and + valueFlowStepSsaEqFlowCond(pos, v, e, delta) + } + /** * Holds if `add` is the addition of `larg` and `rarg`, neither of which are * `ConstantIntegerExpr`s. diff --git a/cpp/ql/lib/semmle/code/cpp/rangeanalysis/new/internal/semantic/analysis/RangeUtils.qll b/cpp/ql/lib/semmle/code/cpp/rangeanalysis/new/internal/semantic/analysis/RangeUtils.qll index 52377c5d14a..1b5da03feec 100644 --- a/cpp/ql/lib/semmle/code/cpp/rangeanalysis/new/internal/semantic/analysis/RangeUtils.qll +++ b/cpp/ql/lib/semmle/code/cpp/rangeanalysis/new/internal/semantic/analysis/RangeUtils.qll @@ -49,6 +49,7 @@ module RangeUtil Lang> implements Range::Ut * - `isEq = true` : `v == e + delta` * - `isEq = false` : `v != e + delta` */ + pragma[nomagic] SemGuard semEqFlowCond( SemSsaVariable v, SemExpr e, D::Delta delta, boolean isEq, boolean testIsTrue ) { From bdad9e197bcf7be73d4e121e6d297c5345295398 Mon Sep 17 00:00:00 2001 From: Mathias Vorreiter Pedersen Date: Thu, 24 Aug 2023 15:57:39 +0100 Subject: [PATCH 047/122] C++: Fix more joins. Before: ``` [2023-08-24 15:47:20] Evaluated non-recursive predicate _IRBlock#896e97af::IRBlock::dominates#1#dispred#ff_Ssa#da392372::Make#SsaInternals#7b362d2f::SsaInpu__#antijoin_rhs@94b1847k in 9831ms (size: 70660). Evaluated relational algebra for predicate _IRBlock#896e97af::IRBlock::dominates#1#dispred#ff_Ssa#da392372::Make#SsaInternals#7b362d2f::SsaInpu__#antijoin_rhs@94b1847k with tuple counts: 1121232 ~0% {4} r1 = JOIN _DataFlowUtil#47741e1f::Cached::simpleLocalFlowStep#2#ff_10#join_rhs_DataFlowUtil#47741e1f::TSsaPhiN__#shared WITH Ssa#da392372::Make#SsaInternals#7b362d2f::SsaInput#::DefinitionExt::definesAt#4#dispred#fffff ON FIRST 1 OUTPUT Rhs.2, Lhs.0, Lhs.1, Lhs.2 265759166 ~2% {4} r2 = JOIN r1 WITH IRBlock#896e97af::IRBlock::dominates#1#dispred#ff ON FIRST 1 OUTPUT Lhs.3, Rhs.1, Lhs.1, Lhs.2 70684 ~5% {3} r3 = JOIN r2 WITH project#DataFlowUtil#47741e1f::Node::hasIndexInBlock#fff ON FIRST 2 OUTPUT Lhs.2, Lhs.3, Lhs.0 return r3 [2023-08-24 15:47:29] Evaluated non-recursive predicate DataFlowUtil#47741e1f::SsaPhiNode::getAnInput#1#dispred#fff@b6f296tl in 8943ms (size: 1121232). Evaluated relational algebra for predicate DataFlowUtil#47741e1f::SsaPhiNode::getAnInput#1#dispred#fff@b6f296tl with tuple counts: 1050572 ~2% {3} r1 = _DataFlowUtil#47741e1f::Cached::simpleLocalFlowStep#2#ff_10#join_rhs_DataFlowUtil#47741e1f::TSsaPhiN__#shared AND NOT _IRBlock#896e97af::IRBlock::dominates#1#dispred#ff_Ssa#da392372::Make#SsaInternals#7b362d2f::SsaInpu__#antijoin_rhs(Lhs.0, Lhs.1, Lhs.2) 1050572 ~3% {3} r2 = SCAN r1 OUTPUT In.1, false, In.2 1121232 ~0% {3} r3 = JOIN _DataFlowUtil#47741e1f::Cached::simpleLocalFlowStep#2#ff_10#join_rhs_DataFlowUtil#47741e1f::TSsaPhiN__#shared WITH Ssa#da392372::Make#SsaInternals#7b362d2f::SsaInput#::DefinitionExt::definesAt#4#dispred#fffff ON FIRST 1 OUTPUT Rhs.2, Lhs.1, Lhs.2 265759166 ~1% {3} r4 = JOIN r3 WITH IRBlock#896e97af::IRBlock::dominates#1#dispred#ff ON FIRST 1 OUTPUT Lhs.2, Rhs.1, Lhs.1 70684 ~0% {2} r5 = JOIN r4 WITH project#DataFlowUtil#47741e1f::Node::hasIndexInBlock#fff ON FIRST 2 OUTPUT Lhs.2, Lhs.0 70684 ~0% {3} r6 = SCAN r5 OUTPUT In.0, true, In.1 1121256 ~2% {3} r7 = r2 UNION r6 return r7 ``` After: ``` Evaluated non-recursive predicate DataFlowUtil#47741e1f::SsaPhiNode::getAnInput#1#dispred#fff@59ab2a2e in 456ms (size: 1117096). Evaluated relational algebra for predicate DataFlowUtil#47741e1f::SsaPhiNode::getAnInput#1#dispred#fff@59ab2a2e with tuple counts: 384518 ~0% {2} r1 = JOIN DataFlowUtil#47741e1f::TSsaPhiNode#ff WITH Ssa#da392372::Make#SsaInternals#7b362d2f::SsaInput#::DefinitionExt::definesAt#4#dispred#fffff ON FIRST 1 OUTPUT Lhs.1, Rhs.2 1121232 ~0% {3} r2 = JOIN r1 WITH DataFlowUtil#47741e1f::Cached::simpleLocalFlowStep#2#ff_10#join_rhs ON FIRST 1 OUTPUT Rhs.1, Lhs.0, Lhs.1 1117447 ~0% {4} r3 = JOIN r2 WITH project#DataFlowUtil#47741e1f::Node::hasIndexInBlock#fff ON FIRST 1 OUTPUT Lhs.2, Rhs.1, Lhs.1, Lhs.0 70684 ~0% {2} r4 = JOIN r3 WITH IRBlock#896e97af::IRBlock::dominates#1#dispred#ff ON FIRST 2 OUTPUT Lhs.2, Lhs.3 70684 ~0% {3} r5 = SCAN r4 OUTPUT In.0, true, In.1 1117447 ~0% {4} r6 = JOIN r2 WITH project#DataFlowUtil#47741e1f::Node::hasIndexInBlock#fff ON FIRST 1 OUTPUT Lhs.2, Rhs.1, Lhs.1, Lhs.0 {4} r7 = r6 AND NOT IRBlock#896e97af::IRBlock::dominates#1#dispred#ff(Lhs.0, Lhs.1) 1046763 ~0% {2} r8 = SCAN r7 OUTPUT In.2, In.3 1046763 ~3% {3} r9 = SCAN r8 OUTPUT In.0, false, In.1 1117447 ~2% {3} r10 = r5 UNION r9 return r10 ``` --- .../semmle/code/cpp/ir/dataflow/internal/DataFlowUtil.qll | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowUtil.qll b/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowUtil.qll index 42198d12372..9355493303a 100644 --- a/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowUtil.qll +++ b/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowUtil.qll @@ -553,9 +553,11 @@ class SsaPhiNode extends Node, TSsaPhiNode { cached final Node getAnInput(boolean fromBackEdge) { localFlowStep(result, this) and - if phi.getBasicBlock().dominates(result.getBasicBlock()) - then fromBackEdge = true - else fromBackEdge = false + exists(IRBlock bPhi, IRBlock bResult | + bPhi = phi.getBasicBlock() and bResult = result.getBasicBlock() + | + if bPhi.dominates(bResult) then fromBackEdge = true else fromBackEdge = false + ) } /** Gets a node that is used as input to this phi node. */ From 242a49e6f1838c73f989db1a0e53a5b82ee51f2c Mon Sep 17 00:00:00 2001 From: Geoffrey White <40627776+geoffw0@users.noreply.github.com> Date: Thu, 24 Aug 2023 17:19:01 +0100 Subject: [PATCH 048/122] Apply suggestions from code review Co-authored-by: mc <42146119+mchammer01@users.noreply.github.com> --- .../change-notes/2023-08-23-incomplete-hostname-regex.md | 2 +- .../queries/Security/CWE-020/IncompleteHostnameRegex.qhelp | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/swift/ql/src/change-notes/2023-08-23-incomplete-hostname-regex.md b/swift/ql/src/change-notes/2023-08-23-incomplete-hostname-regex.md index 98daf65253d..d70dfce16f7 100644 --- a/swift/ql/src/change-notes/2023-08-23-incomplete-hostname-regex.md +++ b/swift/ql/src/change-notes/2023-08-23-incomplete-hostname-regex.md @@ -2,4 +2,4 @@ category: newQuery --- -* Added new query "Incomplete regular expression for hostnames" (`swift/incomplete-hostname-regexp`). This query finds regular expressions matching a URL or hostname that might match more hostnames than expected. +* Added new query "Incomplete regular expression for hostnames" (`swift/incomplete-hostname-regexp`). This query finds regular expressions matching a URL or hostname that may match more hostnames than expected. diff --git a/swift/ql/src/queries/Security/CWE-020/IncompleteHostnameRegex.qhelp b/swift/ql/src/queries/Security/CWE-020/IncompleteHostnameRegex.qhelp index 803e8ddbe08..ef374fc9752 100644 --- a/swift/ql/src/queries/Security/CWE-020/IncompleteHostnameRegex.qhelp +++ b/swift/ql/src/queries/Security/CWE-020/IncompleteHostnameRegex.qhelp @@ -50,7 +50,7 @@

    - The check is however easy to bypass because the unescaped + The check is, however, easy to bypass because the unescaped . allows for any character before example.com, effectively allowing the redirect to go to an attacker-controlled domain such as wwwXexample.com. @@ -68,7 +68,7 @@ -

  • OWASP: Server Side Request Forgery
    • -
  • OWASP: Unvalidated Redirects and Forwards Cheat Sheet
    • +
  • OWASP: Server Side Request Forgery.
    • +
  • OWASP: Unvalidated Redirects and Forwards Cheat Sheet.
    • From 415d9e0674427794b7656c3ca2d2737c054a89c3 Mon Sep 17 00:00:00 2001 From: Geoffrey White <40627776+geoffw0@users.noreply.github.com> Date: Thu, 24 Aug 2023 18:41:48 +0100 Subject: [PATCH 049/122] Swift: Address review comments. --- .../ql/src/queries/Security/CWE-020/IncompleteHostnameRegex.ql | 2 +- .../queries/Security/CWE-020/IncompleteHostnameRegexBad.swift | 2 +- .../queries/Security/CWE-020/IncompleteHostnameRegexGood.swift | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/swift/ql/src/queries/Security/CWE-020/IncompleteHostnameRegex.ql b/swift/ql/src/queries/Security/CWE-020/IncompleteHostnameRegex.ql index a9a49720e58..030d4fbf19d 100644 --- a/swift/ql/src/queries/Security/CWE-020/IncompleteHostnameRegex.ql +++ b/swift/ql/src/queries/Security/CWE-020/IncompleteHostnameRegex.ql @@ -1,6 +1,6 @@ /** * @name Incomplete regular expression for hostnames - * @description Matching a URL or hostname against a regular expression that contains an unescaped dot as part of the hostname might match more hostnames than expected. + * @description Matching a URL or hostname against a regular expression that contains an unescaped dot as part of the hostname may match more hostnames than expected. * @kind problem * @problem.severity warning * @security-severity 7.8 diff --git a/swift/ql/src/queries/Security/CWE-020/IncompleteHostnameRegexBad.swift b/swift/ql/src/queries/Security/CWE-020/IncompleteHostnameRegexBad.swift index c43294fdb23..3e28022ab98 100644 --- a/swift/ql/src/queries/Security/CWE-020/IncompleteHostnameRegexBad.swift +++ b/swift/ql/src/queries/Security/CWE-020/IncompleteHostnameRegexBad.swift @@ -5,7 +5,7 @@ func handleUrl(_ urlString: String) { let redirectParam = components?.queryItems?.first(where: { $0.name == "url" }) // check we trust the host - let regex = #/^(www|beta).example.com//# + let regex = #/^(www|beta).example.com//# // BAD if let match = redirectParam?.value?.firstMatch(of: regex) { // ... trust the URL ... } diff --git a/swift/ql/src/queries/Security/CWE-020/IncompleteHostnameRegexGood.swift b/swift/ql/src/queries/Security/CWE-020/IncompleteHostnameRegexGood.swift index 082206836dc..fad4135a263 100644 --- a/swift/ql/src/queries/Security/CWE-020/IncompleteHostnameRegexGood.swift +++ b/swift/ql/src/queries/Security/CWE-020/IncompleteHostnameRegexGood.swift @@ -5,7 +5,7 @@ func handleUrl(_ urlString: String) { let redirectParam = components?.queryItems?.first(where: { $0.name == "url" }) // check we trust the host - let regex = #/^(www|beta)\.example\.com//# + let regex = #/^(www|beta)\.example\.com//# // GOOD if let match = redirectParam?.value?.firstMatch(of: regex) { // ... trust the URL ... } From 7ad1a21c2dafafc54ba82de5b37dc12b84dc2320 Mon Sep 17 00:00:00 2001 From: Rasmus Lerchedahl Petersen Date: Tue, 15 Aug 2023 21:23:50 +0200 Subject: [PATCH 050/122] Python: make mode characters not be characters They are simply considered part of the group start. --- .../python/regexp/internal/ParseRegExp.qll | 33 +++++++++++++++---- .../Security/CWE-730-ReDoS/ReDoS.expected | 4 +-- 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/python/ql/lib/semmle/python/regexp/internal/ParseRegExp.qll b/python/ql/lib/semmle/python/regexp/internal/ParseRegExp.qll index 52fa85ded56..b4383ff58ff 100644 --- a/python/ql/lib/semmle/python/regexp/internal/ParseRegExp.qll +++ b/python/ql/lib/semmle/python/regexp/internal/ParseRegExp.qll @@ -683,12 +683,34 @@ class RegExp extends Expr instanceof StrConst { * Holds if a parse mode starts between `start` and `end`. */ private predicate flag_group_start(int start, int end) { + exists(int no_modes_end | + this.flag_group_start_no_modes(start, no_modes_end) and + end = max(int i | this.mode_character(start, i) | i + 1) + ) + } + + /** + * Holds if the initial part of a parse mode, not containing any + * mode characters is between `start` and `end`. + */ + private predicate flag_group_start_no_modes(int start, int end) { this.isGroupStart(start) and this.getChar(start + 1) = "?" and this.getChar(start + 2) in ["i", "L", "m", "s", "u", "x"] and end = start + 2 } + /** + * Holds if `pos` contains a mo character from the + * flag group starting at `start`. + */ + private predicate mode_character(int start, int pos) { + this.flag_group_start_no_modes(start, pos) + or + this.mode_character(start, pos - 1) and + this.getChar(pos) in ["i", "L", "m", "s", "u", "x"] + } + /** * Holds if a parse mode group is between `start` and `end`, and includes the * mode flag `c`. For example the following span, with mode flag `i`: @@ -696,11 +718,10 @@ class RegExp extends Expr instanceof StrConst { * (?i) * ``` */ - private predicate flag_group(int start, int end, string c) { - exists(int inStart, int inEnd | - this.flag_group_start(start, inStart) and - this.groupContents(start, end, inStart, inEnd) and - this.getChar([inStart .. inEnd - 1]) = c + private predicate flag(string c) { + exists(int pos | + this.mode_character(_, pos) and + this.getChar(pos) = c ) } @@ -709,7 +730,7 @@ class RegExp extends Expr instanceof StrConst { * it is defined by a prefix. */ string getModeFromPrefix() { - exists(string c | this.flag_group(_, _, c) | + exists(string c | this.flag(c) | c = "i" and result = "IGNORECASE" or c = "L" and result = "LOCALE" diff --git a/python/ql/test/query-tests/Security/CWE-730-ReDoS/ReDoS.expected b/python/ql/test/query-tests/Security/CWE-730-ReDoS/ReDoS.expected index 0c2dccbd2d4..051fd9ccc5f 100644 --- a/python/ql/test/query-tests/Security/CWE-730-ReDoS/ReDoS.expected +++ b/python/ql/test/query-tests/Security/CWE-730-ReDoS/ReDoS.expected @@ -105,5 +105,5 @@ | redos.py:391:15:391:25 | (\\u0061\|a)* | This part of the regular expression may cause exponential backtracking on strings starting with 'X' and containing many repetitions of 'a'. | | unittests.py:5:17:5:23 | (\u00c6\|\\\u00c6)+ | This part of the regular expression may cause exponential backtracking on strings starting with 'X' and containing many repetitions of '\\u00c6'. | | unittests.py:9:16:9:24 | (?:.\|\\n)* | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of '\\n'. | -| unittests.py:11:20:11:28 | (?:.\|\\n)* | This part of the regular expression may cause exponential backtracking on strings starting with 's' and containing many repetitions of '\\n'. | -| unittests.py:12:21:12:29 | (?:.\|\\n)* | This part of the regular expression may cause exponential backtracking on strings starting with 'is' and containing many repetitions of '\\n'. | +| unittests.py:11:20:11:28 | (?:.\|\\n)* | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of '\\n'. | +| unittests.py:12:21:12:29 | (?:.\|\\n)* | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of '\\n'. | From 88fc96e8d74593c42934a12c284f27d2b65809c3 Mon Sep 17 00:00:00 2001 From: Rasmus Lerchedahl Petersen Date: Wed, 16 Aug 2023 10:56:32 +0200 Subject: [PATCH 051/122] Python: Add test with prefix --- python/ql/test/query-tests/Security/CWE-730-ReDoS/unittests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/ql/test/query-tests/Security/CWE-730-ReDoS/unittests.py b/python/ql/test/query-tests/Security/CWE-730-ReDoS/unittests.py index 0a49b8a52a9..4106a691558 100644 --- a/python/ql/test/query-tests/Security/CWE-730-ReDoS/unittests.py +++ b/python/ql/test/query-tests/Security/CWE-730-ReDoS/unittests.py @@ -10,3 +10,4 @@ re.compile(r'(?:.|\n)*b', re.DOTALL) # Has ReDoS. re.compile(r'(?i)(?:.|\n)*b') # No ReDoS. re.compile(r'(?s)(?:.|\n)*b') # Has ReDoS. re.compile(r'(?is)(?:.|\n)*b') # Has ReDoS. +re.compile(r'(?is)X(?:.|\n)*Y') # Has ReDoS. From e9e6bce80a7a33924742cc3f9373accb26ea0c4b Mon Sep 17 00:00:00 2001 From: Rasmus Lerchedahl Petersen Date: Wed, 16 Aug 2023 11:49:23 +0200 Subject: [PATCH 052/122] shared: handle empty groups in delta --- shared/regex/codeql/regex/nfa/NfaUtils.qll | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/shared/regex/codeql/regex/nfa/NfaUtils.qll b/shared/regex/codeql/regex/nfa/NfaUtils.qll index a951820bda1..2b14fe28aca 100644 --- a/shared/regex/codeql/regex/nfa/NfaUtils.qll +++ b/shared/regex/codeql/regex/nfa/NfaUtils.qll @@ -760,6 +760,12 @@ module Make { or exists(RegExpGroup grp | lbl = Epsilon() | q1 = before(grp) and q2 = before(grp.getChild(0))) or + exists(RegExpGroup grp | lbl = Epsilon() | + not exists(grp.getAChild()) and + q1 = before(grp) and + q2 = before(grp.getSuccessor()) + ) + or exists(EffectivelyStar star | lbl = Epsilon() | q1 = before(star) and q2 = before(star.getChild(0)) or From d3c24ba11093d89fa5bee293b63c9008cd939340 Mon Sep 17 00:00:00 2001 From: Rasmus Lerchedahl Petersen Date: Wed, 16 Aug 2023 13:38:10 +0200 Subject: [PATCH 053/122] =?UTF-8?q?Python=C3=86=20fix=20test=20expectation?= =?UTF-8?q?s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Security/CWE-116-BadTagFilter/BadTagFilter.expected | 1 + .../ql/test/query-tests/Security/CWE-730-ReDoS/ReDoS.expected | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/python/ql/test/query-tests/Security/CWE-116-BadTagFilter/BadTagFilter.expected b/python/ql/test/query-tests/Security/CWE-116-BadTagFilter/BadTagFilter.expected index cc9da9cfdc8..407a5490e8c 100644 --- a/python/ql/test/query-tests/Security/CWE-116-BadTagFilter/BadTagFilter.expected +++ b/python/ql/test/query-tests/Security/CWE-116-BadTagFilter/BadTagFilter.expected @@ -1,6 +1,7 @@ | tst.py:4:20:4:43 | .*?<\\/script> | This regular expression does not match script end tags like . | | tst.py:5:20:5:43 | .*?<\\/script> | This regular expression does not match script end tags like . | | tst.py:9:20:9:30 |