Fix test results

java/ql/lotsofmodels.py

@@ -0,0 +1,33 @@
+# Tool to run queries from different packs each containing lots of data extensions.
+# Usage:
+# python3 lotsofmodels.py <path to codeql repo>
+
+import os
+import subprocess
+import sys
+import tempfile
+
+# Add Models as Data script directory to sys.path.
+gitroot = subprocess.check_output(["git", "rev-parse", "--show-toplevel"]).decode("utf-8").strip()
+madpath = os.path.join(gitroot, "misc/scripts/models-as-data/")
+sys.path.append(madpath)
+
+import helpers
+import convert_extensions as extensions
+
+print('Executing script to run dummy queries that loads lots of extensions files.')
+print('Making a dummy database.')
+
+# Configuration
+workDir = tempfile.mkdtemp()
+projectDir = os.path.join(workDir, "project")
+emptyFile = os.path.join(workDir, "Empty.java") 
+dbDir = os.path.join(workDir, "db")
+
+# Make dummy project
+with open(emptyFile, "w") as f:
+    f.write("class Empty {}")
+helpers.run_cmd(['codeql', 'database', 'create', f'--language=java', '-c', f'javac {emptyFile}', dbDir], "Failed to create dummy database.")
+
+# Run dummy queries
+helpers.run_cmd (['codeql', 'database', 'analyze', '--format=csv', '--output=myoutput', '-vvvv', '--additional-packs', sys.argv[1], '--', dbDir, 'p1/p1.ql', 'p2/p2.ql', 'p3/p3.ql', 'p4/p4.ql', 'p5/p5.ql'])

java/ql/p1/p1.ql

@@ -0,0 +1,10 @@
+/**
+ * @name p1
+ * @description p1
+ * @kind diagnostic
+ * @id java/p1
+ */
+
+import java
+
+select "p1"

java/ql/p1/qlpack.yml

@@ -0,0 +1,7 @@
+name: codeql/java-queries-p1
+version: 0.0.0
+extractor: java
+dependencies:
+  codeql/java-all: '*'
+dataExtensions:
+  - ext/*.model.yml

java/ql/p2/p2.ql

@@ -0,0 +1,10 @@
+/**
+ * @name p2
+ * @description p2
+ * @kind diagnostic
+ * @id java/p2
+ */
+
+import java
+
+select "p2"

java/ql/p2/qlpack.yml

@@ -0,0 +1,7 @@
+name: codeql/java-queries-p2
+version: 0.0.0
+extractor: java
+dependencies:
+  codeql/java-all: '*'
+dataExtensions:
+  - ext/*.model.yml

java/ql/p3/p3.ql

@@ -0,0 +1,10 @@
+/**
+ * @name p3
+ * @description p3
+ * @kind diagnostic
+ * @id java/p3
+ */
+
+import java
+
+select "p3"

java/ql/p3/qlpack.yml

@@ -0,0 +1,7 @@
+name: codeql/java-queries-p3
+version: 0.0.0
+extractor: java
+dependencies:
+  codeql/java-all: '*'
+dataExtensions:
+  - ext/*.model.yml

java/ql/p4/p4.ql

@@ -0,0 +1,10 @@
+/**
+ * @name p4
+ * @description p4
+ * @kind diagnostic
+ * @id java/p4
+ */
+
+import java
+
+select "p4"

java/ql/p4/qlpack.yml

@@ -0,0 +1,7 @@
+name: codeql/java-queries-p4
+version: 0.0.0
+extractor: java
+dependencies:
+  codeql/java-all: '*'
+dataExtensions:
+  - ext/*.model.yml

java/ql/p5/p5.ql

@@ -0,0 +1,10 @@
+/**
+ * @name p5
+ * @description p5
+ * @kind diagnostic
+ * @id java/p5
+ */
+
+import java
+
+select "p5"

java/ql/p5/qlpack.yml

@@ -0,0 +1,7 @@
+name: codeql/java-queries-p5
+version: 0.0.0
+extractor: java
+dependencies:
+  codeql/java-all: '*'
+dataExtensions:
+  - ext/*.model.yml

java/ql/src/utils/lotsofmodels/LotsOfModels.py

@@ -0,0 +1,73 @@
+#!/usr/bin/python3
+
+import json
+import os
+import os.path
+import shlex
+import subprocess
+import sys
+import tempfile
+
+# Add Model as Data script directory to sys.path.
+gitroot = subprocess.check_output(["git", "rev-parse", "--show-toplevel"]).decode("utf-8").strip()
+madpath = os.path.join(gitroot, "misc/scripts/models-as-data/")
+sys.path.append(madpath)
+
+import helpers
+
+def quote_if_needed(v):
+    # string columns
+    if type(v) is str:
+        return "\"" + v + "\""
+    # bool column
+    return str(v)
+
+def insert_update(rows, key, value):
+    if key in rows:
+        rows[key] += value
+    else:
+        rows[key] = value
+
+def parseData(data):
+    rows = { }
+    for row in data:
+        d = map(quote_if_needed, row)
+        insert_update(rows, row[0], "      - [" + ', '.join(d) + ']\n')
+
+    return rows
+
+class Generator:
+    def __init__(self, language, n):
+        self.language = language
+        self.codeQlRoot = subprocess.check_output(["git", "rev-parse", "--show-toplevel"]).decode("utf-8").strip()
+        self.extDir = os.path.join(self.codeQlRoot, f"{self.language}/ql/lib/ext/")
+        self.modelFileExtension = ".model.yml"
+        self.packagecount = n
+
+
+    def makeData(self, f, package):
+        n = 30
+        for a in range(n):
+            for b in range(n):
+                for c in range(n):
+                    for k in ["value", "taint"]:
+                        for p in ["manual", "generated"]:
+                            f.write(f"""      - ["{package}", "myClass{str(a)}", False, "myName{str(b)}", "(Object)", "", "Argument[{str(c)}]", "ReturnValue", "{k}", "{p}"]""")
+                            f.write("\n")
+
+
+    def run(self):
+        for i in range(self.packagecount):
+            package = f"package{i}"
+            with open(self.extDir + "/" + package + self.modelFileExtension, "w") as f:
+                f.write("extensions:\n")
+                f.write("  - addsTo:\n")
+                f.write("      pack: codeql/java-all\n")
+                f.write("      extensible: summaryModel\n")
+                f.write("    data:\n")
+                self.makeData(f, package)
+
+
+## Make Lots of Models
+language = "java"
+Generator(language, int(sys.argv[1])).run()

java/ql/test/library-tests/dataflow/external-models/dummymodels.expected

@@ -0,0 +1,155 @@
+|  |
+| android.app |
+| android.content |
+| android.database |
+| android.database.sqlite |
+| android.net |
+| android.os |
+| android.widget |
+| androidx.core.app |
+| androidx.slice.builders |
+| cn.hutool.core.codec |
+| com.esotericsoftware.kryo5.io |
+| com.esotericsoftware.kryo.io |
+| com.fasterxml.jackson.core |
+| com.fasterxml.jackson.databind |
+| com.google.common.base |
+| com.google.common.cache |
+| com.google.common.collect |
+| com.google.common.io |
+| com.rabbitmq.client |
+| com.rabbitmq.client.impl |
+| flexjson |
+| generatedtest |
+| jakarta.json |
+| jakarta.json.stream |
+| jakarta.ws.rs.core |
+| java.beans |
+| java.io |
+| java.lang |
+| java.net |
+| java.nio |
+| java.nio.channels |
+| java.nio.file |
+| java.util |
+| java.util.concurrent |
+| java.util.logging |
+| java.util.regex |
+| java.util.stream |
+| java.util.zip |
+| javax.jms |
+| javax.json |
+| javax.json.stream |
+| javax.servlet.http |
+| javax.ws.rs.core |
+| javax.xml.transform.sax |
+| javax.xml.transform.stream |
+| jodd.json |
+| kotlin |
+| kotlin.collections |
+| kotlin.comparisons |
+| kotlin.coroutines |
+| kotlin.coroutines.intrinsics |
+| kotlin.coroutines.jvm.internal |
+| kotlin.io |
+| kotlin.jvm.internal |
+| kotlin.properties |
+| kotlin.random |
+| kotlin.ranges |
+| kotlin.reflect |
+| kotlin.sequences |
+| kotlin.text |
+| kotlin.time |
+| my.callback.qltest |
+| my.qltest |
+| my.qltest.synth |
+| okhttp3 |
+| org.apache.commons.codec |
+| org.apache.commons.collections |
+| org.apache.commons.collections4 |
+| org.apache.commons.collections4.bag |
+| org.apache.commons.collections4.bidimap |
+| org.apache.commons.collections4.collection |
+| org.apache.commons.collections4.iterators |
+| org.apache.commons.collections4.keyvalue |
+| org.apache.commons.collections4.list |
+| org.apache.commons.collections4.map |
+| org.apache.commons.collections4.multimap |
+| org.apache.commons.collections4.multiset |
+| org.apache.commons.collections4.properties |
+| org.apache.commons.collections4.queue |
+| org.apache.commons.collections4.set |
+| org.apache.commons.collections4.splitmap |
+| org.apache.commons.collections4.trie |
+| org.apache.commons.collections.bag |
+| org.apache.commons.collections.bidimap |
+| org.apache.commons.collections.collection |
+| org.apache.commons.collections.iterators |
+| org.apache.commons.collections.keyvalue |
+| org.apache.commons.collections.list |
+| org.apache.commons.collections.map |
+| org.apache.commons.collections.multimap |
+| org.apache.commons.collections.multiset |
+| org.apache.commons.collections.properties |
+| org.apache.commons.collections.queue |
+| org.apache.commons.collections.set |
+| org.apache.commons.collections.splitmap |
+| org.apache.commons.collections.trie |
+| org.apache.commons.io |
+| org.apache.commons.io.charset |
+| org.apache.commons.io.comparator |
+| org.apache.commons.io.file |
+| org.apache.commons.io.file.spi |
+| org.apache.commons.io.filefilter |
+| org.apache.commons.io.input |
+| org.apache.commons.io.input.buffer |
+| org.apache.commons.io.monitor |
+| org.apache.commons.io.output |
+| org.apache.commons.io.serialization |
+| org.apache.commons.lang3 |
+| org.apache.commons.lang3.builder |
+| org.apache.commons.lang3.mutable |
+| org.apache.commons.lang3.text |
+| org.apache.commons.lang3.tuple |
+| org.apache.commons.text |
+| org.apache.commons.text.lookup |
+| org.apache.hc.core5.function |
+| org.apache.hc.core5.http |
+| org.apache.hc.core5.http.io.entity |
+| org.apache.hc.core5.http.message |
+| org.apache.hc.core5.net |
+| org.apache.hc.core5.util |
+| org.apache.http |
+| org.apache.http.entity |
+| org.apache.http.message |
+| org.apache.http.params |
+| org.apache.http.util |
+| org.apache.ibatis.jdbc |
+| org.apache.logging.log4j |
+| org.apache.shiro.codec |
+| org.json |
+| org.slf4j.spi |
+| org.springframework.beans |
+| org.springframework.cache |
+| org.springframework.context |
+| org.springframework.data.repository |
+| org.springframework.http |
+| org.springframework.ui |
+| org.springframework.util |
+| org.springframework.validation |
+| org.springframework.web.multipart |
+| org.springframework.web.util |
+| org.thymeleaf |
+| org.xml.sax |
+| package0 |
+| package1 |
+| package2 |
+| ratpack.core.form |
+| ratpack.core.handling |
+| ratpack.core.http |
+| ratpack.exec |
+| ratpack.form |
+| ratpack.func |
+| ratpack.handling |
+| ratpack.http |
+| ratpack.util |

java/ql/test/library-tests/dataflow/external-models/dummymodels.ql

@@ -0,0 +1,6 @@
+import java
+import semmle.code.java.dataflow.ExternalFlowExtensions
+
+from string package
+where summaryModel(package, _, _, _, _, _, _, _, _, _)
+select package