Debug: removed grep

Co-authored-by: Arthur Baars <aibaars@github.com>

.github/workflows/internal-ci-checks.yml

@@ -0,0 +1,30 @@
+name: Internal CI Checks
+# This workflows checks if the author of a PR is a member of the CodeQL team
+# and adds `ready-for-internal-ci` label to trigger the internal CI
+
+on:
+  pull_request_target:
+
+jobs:
+  set-label:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set a label to trigger internal CI checks
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          USERNAME: ${{ github.event.pull_request.user.login }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+          LABEL: "ready-for-internal-ci"
+        run: |
+          set +eo pipefail
+          echo "Testing API calls"
+          gh api -i /repos/github/codeql
+          echo "Checking if user $USERNAME is a member of the CodeQL team"
+          if [ "$(gh api "/orgs/github/teams/codeql/memberships/$USERNAME" --jq .state)" == "active" ]; then
+              echo "User $USERNAME is a member of the CodeQL team"
+              echo "Adding '${LABEL}' label"
+              gh pr edit "${PR_NUMBER}" --repo "$GITHUB_REPOSITORY" --add-label "$LABEL"
+          else
+            echo "User $USERNAME is not a member of the CodeQL team"
+            echo "To trigger the internal CI, a maintainer needs to add the '${LABEL}' label"
+          fi

.github/workflows/js-ml-tests.yml

@@ -23,9 +23,9 @@ defaults:
     working-directory: javascript/ql/experimental/adaptivethreatmodeling
 
 jobs:
-  qltest:
-    name: Test QL
-    runs-on: ubuntu-latest-xl
+  qlcompile:
+    name: Check QL compilation
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
 
@@ -33,33 +33,36 @@ jobs:
 
       - name: Install pack dependencies
         run: |
-          for pack in modelbuilding src test; do
+          for pack in modelbuilding src; do
             codeql pack install --mode verify -- "${pack}"
           done
-      
-      - name: Cache compilation cache
-        id: query-cache
-        uses: ./.github/actions/cache-query-compilation
-        with: 
-          key: js-ml-test
 
       - name: Check QL compilation
         run: |
           codeql query compile \
             --check-only \
-            --ram 52000 \
+            --ram 5120 \
             --additional-packs "${{ github.workspace }}" \
             --threads=0 \
-            --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" \
             -- \
             lib modelbuilding src
 
+  qltest:
+    name: Run QL tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: ./.github/actions/fetch-codeql
+
+      - name: Install pack dependencies
+        run: codeql pack install -- test
+
       - name: Run QL tests
         run: |
           codeql test run \
             --threads=0 \
-            --ram 52000 \
+            --ram 5120 \
             --additional-packs "${{ github.workspace }}" \
-            --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" \
             -- \
-            test
+            test

csharp/ql/lib/ext/generated/dotnet_runtime.model.yml

@@ -1,5 +1,6 @@
+ 
 # THIS FILE IS AN AUTO-GENERATED MODELS AS DATA FILE. DO NOT EDIT.
-# Definitions of models for the dotnet_runtime framework.
+# Definitions of taint steps in the dotnet_runtime framework.
 
 extensions:
 

go.work

@@ -1,5 +0,0 @@
-go 1.18
-
-use (
-    ./go
-)

go/extractor/extractor.go

@@ -103,29 +103,29 @@ func ExtractWithFlags(buildFlags []string, patterns []string) error {
 	extractUniverseScope()
 	log.Println("Done extracting universe scope.")
 
-	// a map of package path to source directory and module root directory
-	pkgInfos := make(map[string]util.PkgInfo)
+	// a map of package path to package root directory (currently the module root or the source directory)
+	pkgRoots := make(map[string]string)
+	// a map of package path to source code directory
+	pkgDirs := make(map[string]string)
 	// root directories of packages that we want to extract
 	wantedRoots := make(map[string]bool)
 
-	if os.Getenv("CODEQL_EXTRACTOR_GO_FAST_PACKAGE_INFO") != "" {
-		log.Printf("Running go list to resolve package and module directories.")
-		// get all packages information
-		pkgInfos, err = util.GetPkgsInfo(patterns, true, modFlags...)
-		if err != nil {
-			log.Fatalf("Error getting dependency package or module directories: %v.", err)
-		}
-		log.Printf("Done running go list deps: resolved %d packages.", len(pkgInfos))
-	}
-
 	// Do a post-order traversal and extract the package scope of each package
 	packages.Visit(pkgs, func(pkg *packages.Package) bool {
 		return true
 	}, func(pkg *packages.Package) {
 		log.Printf("Processing package %s.", pkg.PkgPath)
 
-		if _, ok := pkgInfos[pkg.PkgPath]; !ok {
-			pkgInfos[pkg.PkgPath] = util.GetPkgInfo(pkg.PkgPath, modFlags...)
+		if _, ok := pkgRoots[pkg.PkgPath]; !ok {
+			mdir := util.GetModDir(pkg.PkgPath, modFlags...)
+			pdir := util.GetPkgDir(pkg.PkgPath, modFlags...)
+			// GetModDir returns the empty string if the module directory cannot be determined, e.g. if the package
+			// is not using modules. If this is the case, fall back to the package directory
+			if mdir == "" {
+				mdir = pdir
+			}
+			pkgRoots[pkg.PkgPath] = mdir
+			pkgDirs[pkg.PkgPath] = pdir
 		}
 
 		log.Printf("Extracting types for package %s.", pkg.PkgPath)
@@ -152,14 +152,11 @@ func ExtractWithFlags(buildFlags []string, patterns []string) error {
 	})
 
 	for _, pkg := range pkgs {
-		pkgInfo, ok := pkgInfos[pkg.PkgPath]
-		if !ok || pkgInfo.PkgDir == "" {
+		if pkgRoots[pkg.PkgPath] == "" {
 			log.Fatalf("Unable to get a source directory for input package %s.", pkg.PkgPath)
 		}
-		wantedRoots[pkgInfo.PkgDir] = true
-		if pkgInfo.ModDir != "" {
-			wantedRoots[pkgInfo.ModDir] = true
-		}
+		wantedRoots[pkgRoots[pkg.PkgPath]] = true
+		wantedRoots[pkgDirs[pkg.PkgPath]] = true
 	}
 
 	log.Println("Done processing dependencies.")
@@ -177,8 +174,7 @@ func ExtractWithFlags(buildFlags []string, patterns []string) error {
 		return true
 	}, func(pkg *packages.Package) {
 		for root, _ := range wantedRoots {
-			pkgInfo := pkgInfos[pkg.PkgPath]
-			relDir, err := filepath.Rel(root, pkgInfo.PkgDir)
+			relDir, err := filepath.Rel(root, pkgDirs[pkg.PkgPath])
 			if err != nil || noExtractRe.MatchString(relDir) {
 				// if the path can't be made relative or matches the noExtract regexp skip it
 				continue
@@ -186,12 +182,8 @@ func ExtractWithFlags(buildFlags []string, patterns []string) error {
 
 			extraction.extractPackage(pkg)
 
-			modDir := pkgInfo.ModDir
-			if modDir == "" {
-				modDir = pkgInfo.PkgDir
-			}
-			if modDir != "" {
-				modPath := filepath.Join(modDir, "go.mod")
+			if pkgRoots[pkg.PkgPath] != "" {
+				modPath := filepath.Join(pkgRoots[pkg.PkgPath], "go.mod")
 				if util.FileExists(modPath) {
 					log.Printf("Extracting %s", modPath)
 					start := time.Now()

go/extractor/util/util.go

@@ -1,9 +1,7 @@
 package util
 
 import (
-	"encoding/json"
 	"errors"
-	"io"
 	"log"
 	"os"
 	"os/exec"
@@ -33,13 +31,13 @@ func Getenv(key string, aliases ...string) string {
 
 // runGoList is a helper function for running go list with format `format` and flags `flags` on
 // package `pkgpath`.
-func runGoList(format string, patterns []string, flags ...string) (string, error) {
-	return runGoListWithEnv(format, patterns, nil, flags...)
+func runGoList(format string, pkgpath string, flags ...string) (string, error) {
+	return runGoListWithEnv(format, pkgpath, nil, flags...)
 }
 
-func runGoListWithEnv(format string, patterns []string, additionalEnv []string, flags ...string) (string, error) {
+func runGoListWithEnv(format string, pkgpath string, additionalEnv []string, flags ...string) (string, error) {
 	args := append([]string{"list", "-e", "-f", format}, flags...)
-	args = append(args, patterns...)
+	args = append(args, pkgpath)
 	cmd := exec.Command("go", args...)
 	cmd.Env = append(os.Environ(), additionalEnv...)
 	out, err := cmd.Output()
@@ -56,89 +54,18 @@ func runGoListWithEnv(format string, patterns []string, additionalEnv []string,
 	return strings.TrimSpace(string(out)), nil
 }
 
-// PkgInfo holds package directory and module directory (if any) for a package
-type PkgInfo struct {
-	PkgDir string // the directory directly containing source code of this package
-	ModDir string // the module directory containing this package, empty if not a module
-}
-
-// GetPkgsInfo gets the absolute module and package root directories for the packages matched by the
-// patterns `patterns`. It passes to `go list` the flags specified by `flags`.  If `includingDeps`
-// is true, all dependencies will also be included.
-func GetPkgsInfo(patterns []string, includingDeps bool, flags ...string) (map[string]PkgInfo, error) {
-	// enable module mode so that we can find a module root if it exists, even if go module support is
-	// disabled by a build
-	if includingDeps {
-		// the flag `-deps` causes all dependencies to be retrieved
-		flags = append(flags, "-deps")
-	}
-
-	// using -json overrides -f format
-	output, err := runGoList("", patterns, append(flags, "-json")...)
-	if err != nil {
-		return nil, err
-	}
-
-	// the output of `go list -json` is a stream of json object
-	type goListPkgInfo struct {
-		ImportPath string
-		Dir        string
-		Module     *struct {
-			Dir string
-		}
-	}
-	pkgInfoMapping := make(map[string]PkgInfo)
-	streamDecoder := json.NewDecoder(strings.NewReader(output))
-	for {
-		var pkgInfo goListPkgInfo
-		decErr := streamDecoder.Decode(&pkgInfo)
-		if decErr == io.EOF {
-			break
-		}
-		if decErr != nil {
-			log.Printf("Error decoding output of go list -json: %s", err.Error())
-			return nil, decErr
-		}
-		pkgAbsDir, err := filepath.Abs(pkgInfo.Dir)
-		if err != nil {
-			log.Printf("Unable to make package dir %s absolute: %s", pkgInfo.Dir, err.Error())
-		}
-		var modAbsDir string
-		if pkgInfo.Module != nil {
-			modAbsDir, err = filepath.Abs(pkgInfo.Module.Dir)
-			if err != nil {
-				log.Printf("Unable to make module dir %s absolute: %s", pkgInfo.Module.Dir, err.Error())
-			}
-		}
-		pkgInfoMapping[pkgInfo.ImportPath] = PkgInfo{
-			PkgDir: pkgAbsDir,
-			ModDir: modAbsDir,
-		}
-	}
-	return pkgInfoMapping, nil
-}
-
-// GetPkgInfo fills the package info structure for the specified package path.
-// It passes the `go list` the flags specified by `flags`.
-func GetPkgInfo(pkgpath string, flags ...string) PkgInfo {
-	return PkgInfo{
-		PkgDir: GetPkgDir(pkgpath, flags...),
-		ModDir: GetModDir(pkgpath, flags...),
-	}
-}
-
 // GetModDir gets the absolute directory of the module containing the package with path
 // `pkgpath`. It passes the `go list` the flags specified by `flags`.
 func GetModDir(pkgpath string, flags ...string) string {
 	// enable module mode so that we can find a module root if it exists, even if go module support is
 	// disabled by a build
-	mod, err := runGoListWithEnv("{{.Module}}", []string{pkgpath}, []string{"GO111MODULE=on"}, flags...)
+	mod, err := runGoListWithEnv("{{.Module}}", pkgpath, []string{"GO111MODULE=on"}, flags...)
 	if err != nil || mod == "<nil>" {
 		// if the command errors or modules aren't being used, return the empty string
 		return ""
 	}
 
-	modDir, err := runGoListWithEnv("{{.Module.Dir}}", []string{pkgpath}, []string{"GO111MODULE=on"}, flags...)
+	modDir, err := runGoListWithEnv("{{.Module.Dir}}", pkgpath, []string{"GO111MODULE=on"}, flags...)
 	if err != nil {
 		return ""
 	}
@@ -154,7 +81,7 @@ func GetModDir(pkgpath string, flags ...string) string {
 // GetPkgDir gets the absolute directory containing the package with path `pkgpath`. It passes the
 // `go list` command the flags specified by `flags`.
 func GetPkgDir(pkgpath string, flags ...string) string {
-	pkgDir, err := runGoList("{{.Dir}}", []string{pkgpath}, flags...)
+	pkgDir, err := runGoList("{{.Dir}}", pkgpath, flags...)
 	if err != nil {
 		return ""
 	}
@@ -170,7 +97,7 @@ func GetPkgDir(pkgpath string, flags ...string) string {
 // DepErrors checks there are any errors resolving dependencies for `pkgpath`. It passes the `go
 // list` command the flags specified by `flags`.
 func DepErrors(pkgpath string, flags ...string) bool {
-	out, err := runGoList("{{if .DepsErrors}}{{else}}error{{end}}", []string{pkgpath}, flags...)
+	out, err := runGoList("{{if .DepsErrors}}{{else}}error{{end}}", pkgpath, flags...)
 	if err != nil {
 		// if go list failed, assume dependencies are broken
 		return false

java/ql/lib/ext/generated/org.apache.commons.io.model.yml

@@ -1,5 +1,6 @@
+ 
 # THIS FILE IS AN AUTO-GENERATED MODELS AS DATA FILE. DO NOT EDIT.
-# Definitions of models for the org.apache.commons.io framework.
+# Definitions of taint steps in the org.apache.commons.io framework.
 
 extensions:
   - addsTo:

java/ql/lib/semmle/code/java/Expr.qll

@@ -793,7 +793,7 @@ class CharacterLiteral extends Literal, @characterliteral {
    * the character to an `int`.
    */
   int getCodePointValue() {
-    if this.getLiteral().matches("'\\\\u____'")
+    if this.getLiteral().matches("'\\u____'")
     then result = fromHex(this.getLiteral().substring(3, 7))
     else result.toUnicode() = this.getValue()
   }

java/ql/src/Likely Bugs/Serialization/NonSerializableField.ql

@@ -55,8 +55,6 @@ string nonSerialReason(RefType t) {
 predicate exceptions(Class c, Field f) {
   f.getDeclaringType() = c and
   (
-    c.isCompilerGenerated()
-    or
     // `Serializable` objects with custom `readObject` or `writeObject` methods
     // may write out the "non-serializable" fields in a different way.
     c.declaresMethod("readObject")
@@ -92,7 +90,7 @@ predicate exceptions(Class c, Field f) {
 
 from Class c, Field f, string reason
 where
-  c.fromSource() and
+  c.getFile().isJavaSourceFile() and
   c.getAStrictAncestor() instanceof TypeSerializable and
   f.getDeclaringType() = c and
   not exceptions(c, f) and

java/ql/src/Security/CWE/CWE-022/TaintedPath.java

@@ -16,9 +16,9 @@ public void sendUserFileFixed(Socket sock, String user) {
 	// ...
 	
 	// GOOD: remove all dots and directory delimiters from the filename before using
-	String filename = filenameReader.readLine().replaceAll("\\.", "").replaceAll("/", "");
+	String filename = filenameReader.readLine().replaceAll("\.", "").replaceAll("/", "");
 	BufferedReader fileReader = new BufferedReader(
 			new FileReader("/home/" + user + "/" + filename));
 
 	// ...
-}
+}

java/ql/src/change-notes/2022-11-25-NonSerializableField-kotlin.md

@@ -1,4 +0,0 @@
----
-category: minorAnalysis
----
-* The query `java/non-serializable-field` is now enabled for Kotlin.

java/ql/src/utils/flowtestcasegenerator/FlowTestCase.qll

@@ -12,7 +12,7 @@ private import FlowTestCaseSupportMethods
 
 /**
  * A CSV row to generate tests for. Users should extend this to define which
- * tests to generate. There should already exist summaries for the rows specified here.
+ * tests to generate. Rows specified here should also satisfy `SummaryModelCsv.row`.
  */
 class TargetSummaryModelCsv extends Unit {
   /**
@@ -42,11 +42,11 @@ predicate summaryModelRow(
 }
 
 /**
- * Gets a CSV row for which a test has been requested, but where a summary has not already been defined.
+ * Gets a CSV row for which a test has been requested, but `SummaryModelCsv.row` does not hold of it.
  */
-query string missingSummaryModel() {
+query string missingSummaryModelCsv() {
   any(TargetSummaryModelCsv target).row(result) and
-  not summaryModelRow(_, _, _, _, _, _, _, _, _, _, result)
+  not any(SummaryModelCsv model).row(result)
 }
 
 /**

java/ql/src/utils/flowtestcasegenerator/FlowTestCaseSupportMethods.qll

@@ -97,12 +97,12 @@ abstract class SupportMethod extends string {
   int getPriority() { result = 50 }
 
   /**
-   * Gets the data extension row describing this support method if it is needed to set up the output for this test.
+   * Gets the CSV row describing this support method if it is needed to set up the output for this test.
    *
-   * For example, `newWithMapValue` will propagate a value from `Argument[0]` to `ReturnValue.MapValue`, and `getMapValue`
+   * For example, `newWithMapValue` will propagate a value from `Argument[0]` to `MapValue of ReturnValue`, and `getMapValue`
    * will do the opposite.
    */
-  string getDataExtensionModel() { none() }
+  string getCsvModel() { none() }
 }
 
 /**
@@ -162,11 +162,10 @@ private class DefaultGetMethod extends GetMethod {
     result = "Object get" + contentToken(c) + "Default(Object container) { return null; }"
   }
 
-  override string getDataExtensionModel() {
+  override string getCsvModel() {
     result =
-      "\"generatedtest\", \"Test\", False, \"" + this.getName() +
-        "\", \"(Object)\", \"\", \"Argument[0]." + getComponentSpec(SummaryComponent::content(c)) +
-        "\", \"ReturnValue\", \"value\", \"manual\""
+      "generatedtest;Test;false;" + this.getName() + ";(Object);;Argument[0]." +
+        getComponentSpec(SummaryComponent::content(c)) + ";ReturnValue;value;manual"
   }
 }
 
@@ -359,11 +358,10 @@ private class DefaultGenMethod extends GenMethod {
     result = "Object newWith" + contentToken(c) + "Default(Object element) { return null; }"
   }
 
-  override string getDataExtensionModel() {
+  override string getCsvModel() {
     result =
-      "\"generatedtest\", \"Test\", False, \"" + this.getName() +
-        "\", \"(Object)\", \"\", \"Argument[0]\", \"ReturnValue." +
-        getComponentSpec(SummaryComponent::content(c)) + "\", \"value\", \"manual\""
+      "generatedtest;Test;false;" + this.getName() + ";(Object);;Argument[0];ReturnValue." +
+        getComponentSpec(SummaryComponent::content(c)) + ";value;manual"
   }
 }
 

java/ql/src/utils/flowtestcasegenerator/GenerateFlowTestCase.py

@@ -24,9 +24,7 @@ contain the needed classes.
 
 If --force is present, existing files may be overwritten.
 
-Requirements:
- - `mvn` and `codeql` should both appear on your path.
- - `--additional-packs /path/to/semmle-code/ql` should be added to your `.config/codeql/config` file.
+Requirements: `mvn` and `codeql` should both appear on your path.
 
 After test generation completes, any lines in specsToTest.csv that didn't produce tests are output.
 If this happens, check the spelling of class and method names, and the syntax of input and output specifications.
@@ -54,12 +52,10 @@ except Exception as e:
 
 resultJava = os.path.join(sys.argv[3], "Test.java")
 resultQl = os.path.join(sys.argv[3], "test.ql")
-resultYml = os.path.join(sys.argv[3], "test.model.yml")
-resultPack = os.path.join(sys.argv[3], "qlpack.yml")
 
-if not force and (os.path.exists(resultJava) or os.path.exists(resultQl) or os.path.exists(resultYml) or os.path.exists(resultPack)):
-    print("Won't overwrite existing files '%s', '%s', '%s' or '%s'." %
-          (resultJava, resultQl, resultYml, resultPack), file=sys.stderr)
+if not force and (os.path.exists(resultJava) or os.path.exists(resultQl)):
+    print("Won't overwrite existing files '%s' or '%s'" %
+          (resultJava, resultQl), file=sys.stderr)
     sys.exit(1)
 
 workDir = tempfile.mkdtemp()
@@ -131,13 +127,7 @@ queryDir = os.path.join(workDir, "query")
 os.makedirs(queryDir)
 qlFile = os.path.join(queryDir, "gen.ql")
 with open(os.path.join(queryDir, "qlpack.yml"), "w") as f:
-    f.write("""name: test-generation-query
-version: 0.0.0
-dependencies:
-  codeql/java-all: '*'
-  codeql/java-queries: '*'
-""")
-
+    f.write("name: test-generation-query\nversion: 0.0.0\nlibraryPathDependencies: codeql/java-queries")
 with open(qlFile, "w") as f:
     f.write(
         "import java\nimport utils.flowtestcasegenerator.GenerateFlowTestCase\n\nclass GenRow extends TargetSummaryModelCsv {\n\n\toverride predicate row(string r) {\n\t\tr = [\n")
@@ -173,9 +163,9 @@ def getTuples(queryName, jsonResult, fname):
 with open(generatedJson, "r") as f:
     generateOutput = json.load(f)
     expectedTables = ("getTestCase", "getASupportMethodModel",
-                      "missingSummaryModel", "getAParseFailure", "noTestCaseGenerated")
+                      "missingSummaryModelCsv", "getAParseFailure", "noTestCaseGenerated")
 
-    testCaseRows, supportModelRows, missingSummaryModelRows, parseFailureRows, noTestCaseGeneratedRows = \
+    testCaseRows, supportModelRows, missingSummaryModelCsvRows, parseFailureRows, noTestCaseGeneratedRows = \
         tuple([getTuples(k, generateOutput, generatedJson)
                for k in expectedTables])
 
@@ -192,9 +182,9 @@ with open(generatedJson, "r") as f:
         print("Expected exactly one column in noTestCaseGenerated relation (got: %s)" %
               json.dumps(noTestCaseGeneratedRows), file=sys.stderr)
 
-    if len(missingSummaryModelRows) != 0:
-        print("Tests for some CSV rows were requested that were not in scope (a summary doesn't already exist):\n" +
-              "\n".join(r[0] for r in missingSummaryModelRows))
+    if len(missingSummaryModelCsvRows) != 0:
+        print("Tests for some CSV rows were requested that were not in scope (SummaryModelCsv.row does not hold):\n" +
+              "\n".join(r[0] for r in missingSummaryModelCsvRows))
         sys.exit(1)
     if len(parseFailureRows) != 0:
         print("The following rows failed to generate any test case. Check package, class and method name spelling, and argument and result specifications:\n%s" %
@@ -217,32 +207,11 @@ def copyfile(fromName, toFileHandle):
 
 with open(resultQl, "w") as f:
     copyfile("testHeader.qlfrag", f)
-
-if len(supportModelRows) != 0:
-    # Make a test extension file
-    with open(resultYml, "w") as f:
-        models = "\n".join('      - [%s]' %
-                          modelSpecRow[0].strip() for modelSpecRow in supportModelRows)
-        dataextensions = f"""extensions:
-  - addsTo:
-      pack: codeql/java-tests
-      extensible: extSummaryModel
-    data:
-{models}
-"""
-        f.write(dataextensions)
-    # Make a qlpack file such that the extension will be picked up
-    with open(resultPack, "w") as f:
-        f.write("""name: example-test-pack
-version: 0.0.0
-extractor: java
-dependencies:
-  codeql/java-all: '*'
-  codeql/java-queries: '*'
-  codeql/java-tests: '*'
-dataExtensions:
-  - test.model.yml
-""")
+    if len(supportModelRows) != 0:
+        copyfile("testModelsHeader.qlfrag", f)
+        f.write(", ".join('"%s"' %
+                          modelSpecRow[0].strip() for modelSpecRow in supportModelRows))
+        copyfile("testModelsFooter.qlfrag", f)
 
 # Make an empty .expected file, since this is an inline-exectations test
 with open(os.path.join(sys.argv[3], "test.expected"), "w"):

java/ql/src/utils/flowtestcasegenerator/GenerateFlowTestCase.qll

@@ -13,13 +13,13 @@ private import FlowTestCaseSupportMethods
 private import FlowTestCaseUtils
 
 /**
- * Gets a CSV row for which a test has been requested, and where there exists a summary, but
+ * Gets a CSV row for which a test has been requested, and `SummaryModelCsv.row` does hold, but
  * nonetheless we can't generate a test case for it, indicating we cannot resolve either the callable
  * spec or an input or output spec.
  */
 query string getAParseFailure(string reason) {
   any(TargetSummaryModelCsv target).row(result) and
-  summaryModelRow(_, _, _, _, _, _, _, _, _, _, result) and
+  any(SummaryModelCsv model).row(result) and
   (
     not summaryModelRow(_, _, _, _, _, _, _, _, _, _, result) and
     reason = "row could not be parsed"
@@ -52,7 +52,7 @@ query string getAParseFailure(string reason) {
  */
 query string noTestCaseGenerated() {
   any(TargetSummaryModelCsv target).row(result) and
-  summaryModelRow(_, _, _, _, _, _, _, _, _, _, result) and
+  any(SummaryModelCsv model).row(result) and
   not exists(getAParseFailure(_)) and
   not exists(any(TestCase tc).getATestSnippetForRow(result))
 }
@@ -85,12 +85,12 @@ SupportMethod getASupportMethod() {
 }
 
 /**
- * Returns a data extension specification of the taint-/value-propagation behavior of a test support method (`get` or `newWith` method).
+ * Returns a CSV specification of the taint-/value-propagation behavior of a test support method (`get` or `newWith` method).
  */
-query string getASupportMethodModel() { result = getASupportMethod().getDataExtensionModel() }
+query string getASupportMethodModel() { result = getASupportMethod().getCsvModel() }
 
 /**
- * Gets a Java file body testing all requested Models as Data rows against whatever classes and methods they resolve against.
+ * Gets a Java file body testing all requested CSV rows against whatever classes and methods they resolve against.
  */
 query string getTestCase() {
   result =

java/ql/src/utils/flowtestcasegenerator/testModelsFooter.qlfrag

@@ -0,0 +1,3 @@
+      ]
+  }
+}

java/ql/src/utils/flowtestcasegenerator/testModelsHeader.qlfrag

@@ -0,0 +1,5 @@
+class SummaryModelTest extends SummaryModelCsv {
+  override predicate row(string row) {
+    row =
+      [
+        //"package;type;overrides;name;signature;ext;inputspec;outputspec;kind;provenance"

javascript/ql/experimental/adaptivethreatmodeling/lib/experimental/adaptivethreatmodeling/ATMConfig.qll

@@ -63,9 +63,9 @@ abstract class AtmConfig extends JS::TaintTracking::Configuration {
     // If the list of characteristics includes positive indicators with maximal confidence for this class, then it's a
     // known sink for the class.
     exists(EndpointCharacteristics::EndpointCharacteristic characteristic |
-      characteristic.appliesToEndpoint(sink) and
+      characteristic.getEndpoints(sink) and
       characteristic
-          .hasImplications(this.getASinkEndpointType(), true, characteristic.maximalConfidence())
+          .getImplications(this.getASinkEndpointType(), true, characteristic.maximalConfidence())
     )
   }
 
@@ -99,7 +99,7 @@ abstract class AtmConfig extends JS::TaintTracking::Configuration {
     // have given the endpoint filter characteristics medium confidence, and we exclude endpoints that have a
     // medium-confidence characteristic that indicates that they are not sinks, either in general or for this sink type.
     exists(EndpointCharacteristics::EndpointCharacteristic filter, float confidence |
-      filter.appliesToEndpoint(candidateSink) and
+      filter.getEndpoints(candidateSink) and
       confidence >= filter.mediumConfidence() and
       // TODO: Experiment with excluding all endpoints that have a medium- or high-confidence characteristic that
       // implies they're not sinks, rather than using only medium-confidence characteristics, by deleting the following
@@ -107,10 +107,10 @@ abstract class AtmConfig extends JS::TaintTracking::Configuration {
       confidence < filter.highConfidence() and
       (
         // Exclude endpoints that have a characteristic that implies they're not sinks for _any_ sink type.
-        filter.hasImplications(any(NegativeType negative), true, confidence)
+        filter.getImplications(any(NegativeType negative), true, confidence)
         or
         // Exclude endpoints that have a characteristic that implies they're not sinks for _this particular_ sink type.
-        filter.hasImplications(this.getASinkEndpointType(), false, confidence)
+        filter.getImplications(this.getASinkEndpointType(), false, confidence)
       ) and
       result = filter
     )
@@ -147,9 +147,7 @@ abstract class AtmConfig extends JS::TaintTracking::Configuration {
    * to this ML-boosted configuration, whereas the unboosted base query does not contain this source and sink
    * combination.
    */
-  predicate hasBoostedFlowPath(
-    JS::DataFlow::PathNode source, JS::DataFlow::PathNode sink, float score
-  ) {
+  predicate getAlerts(JS::DataFlow::PathNode source, JS::DataFlow::PathNode sink, float score) {
     this.hasFlowPath(source, sink) and
     not AtmResultsInfo::isFlowLikelyInBaseQuery(source.getNode(), sink.getNode()) and
     score = AtmResultsInfo::getScoreForFlow(source.getNode(), sink.getNode())

javascript/ql/experimental/adaptivethreatmodeling/lib/experimental/adaptivethreatmodeling/EndpointCharacteristics.qll

@@ -65,7 +65,7 @@ abstract class EndpointCharacteristic extends string {
    * Holds for endpoints that have this characteristic. This predicate contains the logic that applies characteristics
    * to the appropriate set of dataflow nodes.
    */
-  abstract predicate appliesToEndpoint(DataFlow::Node n);
+  abstract predicate getEndpoints(DataFlow::Node n);
 
   /**
    * This predicate describes what the characteristic tells us about an endpoint.
@@ -81,7 +81,7 @@ abstract class EndpointCharacteristic extends string {
    * indicator of whether or not the endpoint belongs to the class. A confidence of 1 means that all endpoints with
    * this characteristic definitively do/don't belong to the class.
    */
-  abstract predicate hasImplications(
+  abstract predicate getImplications(
     EndpointType endpointClass, boolean isPositiveIndicator, float confidence
   );
 
@@ -213,9 +213,9 @@ DataFlow::CallNode getALikelyExternalLibraryCall() { result = getACallWithoutCal
 private class DomBasedXssSinkCharacteristic extends EndpointCharacteristic {
   DomBasedXssSinkCharacteristic() { this = "DomBasedXssSink" }
 
-  override predicate appliesToEndpoint(DataFlow::Node n) { n instanceof DomBasedXss::Sink }
+  override predicate getEndpoints(DataFlow::Node n) { n instanceof DomBasedXss::Sink }
 
-  override predicate hasImplications(
+  override predicate getImplications(
     EndpointType endpointClass, boolean isPositiveIndicator, float confidence
   ) {
     endpointClass instanceof XssSinkType and
@@ -231,9 +231,9 @@ private class DomBasedXssSinkCharacteristic extends EndpointCharacteristic {
 private class TaintedPathSinkCharacteristic extends EndpointCharacteristic {
   TaintedPathSinkCharacteristic() { this = "TaintedPathSink" }
 
-  override predicate appliesToEndpoint(DataFlow::Node n) { n instanceof TaintedPath::Sink }
+  override predicate getEndpoints(DataFlow::Node n) { n instanceof TaintedPath::Sink }
 
-  override predicate hasImplications(
+  override predicate getImplications(
     EndpointType endpointClass, boolean isPositiveIndicator, float confidence
   ) {
     endpointClass instanceof TaintedPathSinkType and
@@ -249,9 +249,9 @@ private class TaintedPathSinkCharacteristic extends EndpointCharacteristic {
 private class SqlInjectionSinkCharacteristic extends EndpointCharacteristic {
   SqlInjectionSinkCharacteristic() { this = "SqlInjectionSink" }
 
-  override predicate appliesToEndpoint(DataFlow::Node n) { n instanceof SqlInjection::Sink }
+  override predicate getEndpoints(DataFlow::Node n) { n instanceof SqlInjection::Sink }
 
-  override predicate hasImplications(
+  override predicate getImplications(
     EndpointType endpointClass, boolean isPositiveIndicator, float confidence
   ) {
     endpointClass instanceof SqlInjectionSinkType and
@@ -267,9 +267,9 @@ private class SqlInjectionSinkCharacteristic extends EndpointCharacteristic {
 private class NosqlInjectionSinkCharacteristic extends EndpointCharacteristic {
   NosqlInjectionSinkCharacteristic() { this = "NosqlInjectionSink" }
 
-  override predicate appliesToEndpoint(DataFlow::Node n) { n instanceof NosqlInjection::Sink }
+  override predicate getEndpoints(DataFlow::Node n) { n instanceof NosqlInjection::Sink }
 
-  override predicate hasImplications(
+  override predicate getImplications(
     EndpointType endpointClass, boolean isPositiveIndicator, float confidence
   ) {
     endpointClass instanceof NosqlInjectionSinkType and
@@ -307,7 +307,7 @@ abstract private class NotASinkCharacteristic extends EndpointCharacteristic {
   bindingset[this]
   NotASinkCharacteristic() { any() }
 
-  override predicate hasImplications(
+  override predicate getImplications(
     EndpointType endpointClass, boolean isPositiveIndicator, float confidence
   ) {
     endpointClass instanceof NegativeType and
@@ -326,7 +326,7 @@ abstract class LikelyNotASinkCharacteristic extends EndpointCharacteristic {
   bindingset[this]
   LikelyNotASinkCharacteristic() { any() }
 
-  override predicate hasImplications(
+  override predicate getImplications(
     EndpointType endpointClass, boolean isPositiveIndicator, float confidence
   ) {
     endpointClass instanceof NegativeType and
@@ -339,7 +339,7 @@ private class LodashUnderscoreCharacteristic extends NotASinkCharacteristic,
   OtherModeledArgumentCharacteristic {
   LodashUnderscoreCharacteristic() { this = "LodashUnderscoreArgument" }
 
-  override predicate appliesToEndpoint(DataFlow::Node n) {
+  override predicate getEndpoints(DataFlow::Node n) {
     any(LodashUnderscore::Member m).getACall().getAnArgument() = n
   }
 }
@@ -348,7 +348,7 @@ private class JQueryArgumentCharacteristic extends NotASinkCharacteristic,
   OtherModeledArgumentCharacteristic {
   JQueryArgumentCharacteristic() { this = "JQueryArgument" }
 
-  override predicate appliesToEndpoint(DataFlow::Node n) {
+  override predicate getEndpoints(DataFlow::Node n) {
     any(JQuery::MethodCall m).getAnArgument() = n
   }
 }
@@ -357,7 +357,7 @@ private class ClientRequestCharacteristic extends NotASinkCharacteristic,
   OtherModeledArgumentCharacteristic {
   ClientRequestCharacteristic() { this = "ClientRequest" }
 
-  override predicate appliesToEndpoint(DataFlow::Node n) {
+  override predicate getEndpoints(DataFlow::Node n) {
     exists(ClientRequest r |
       r.getAnArgument() = n or n = r.getUrl() or n = r.getHost() or n = r.getADataNode()
     )
@@ -368,7 +368,7 @@ private class PromiseDefinitionCharacteristic extends NotASinkCharacteristic,
   OtherModeledArgumentCharacteristic {
   PromiseDefinitionCharacteristic() { this = "PromiseDefinition" }
 
-  override predicate appliesToEndpoint(DataFlow::Node n) {
+  override predicate getEndpoints(DataFlow::Node n) {
     exists(PromiseDefinition p |
       n = [p.getResolveParameter(), p.getRejectParameter()].getACall().getAnArgument()
     )
@@ -379,14 +379,14 @@ private class CryptographicKeyCharacteristic extends NotASinkCharacteristic,
   OtherModeledArgumentCharacteristic {
   CryptographicKeyCharacteristic() { this = "CryptographicKey" }
 
-  override predicate appliesToEndpoint(DataFlow::Node n) { n instanceof CryptographicKey }
+  override predicate getEndpoints(DataFlow::Node n) { n instanceof CryptographicKey }
 }
 
 private class CryptographicOperationFlowCharacteristic extends NotASinkCharacteristic,
   OtherModeledArgumentCharacteristic {
   CryptographicOperationFlowCharacteristic() { this = "CryptographicOperationFlow" }
 
-  override predicate appliesToEndpoint(DataFlow::Node n) {
+  override predicate getEndpoints(DataFlow::Node n) {
     any(CryptographicOperation op).getInput() = n
   }
 }
@@ -395,7 +395,7 @@ private class LoggerMethodCharacteristic extends NotASinkCharacteristic,
   OtherModeledArgumentCharacteristic {
   LoggerMethodCharacteristic() { this = "LoggerMethod" }
 
-  override predicate appliesToEndpoint(DataFlow::Node n) {
+  override predicate getEndpoints(DataFlow::Node n) {
     exists(DataFlow::CallNode call | n = call.getAnArgument() |
       call.getCalleeName() = getAStandardLoggerMethodName()
     )
@@ -406,7 +406,7 @@ private class TimeoutCharacteristic extends NotASinkCharacteristic,
   OtherModeledArgumentCharacteristic {
   TimeoutCharacteristic() { this = "Timeout" }
 
-  override predicate appliesToEndpoint(DataFlow::Node n) {
+  override predicate getEndpoints(DataFlow::Node n) {
     exists(DataFlow::CallNode call | n = call.getAnArgument() |
       call.getCalleeName() = ["setTimeout", "clearTimeout"]
     )
@@ -417,7 +417,7 @@ private class ReceiverStorageCharacteristic extends NotASinkCharacteristic,
   OtherModeledArgumentCharacteristic {
   ReceiverStorageCharacteristic() { this = "ReceiverStorage" }
 
-  override predicate appliesToEndpoint(DataFlow::Node n) {
+  override predicate getEndpoints(DataFlow::Node n) {
     exists(DataFlow::CallNode call | n = call.getAnArgument() |
       call.getReceiver() = DataFlow::globalVarRef(["localStorage", "sessionStorage"])
     )
@@ -428,7 +428,7 @@ private class StringStartsWithCharacteristic extends NotASinkCharacteristic,
   OtherModeledArgumentCharacteristic {
   StringStartsWithCharacteristic() { this = "StringStartsWith" }
 
-  override predicate appliesToEndpoint(DataFlow::Node n) {
+  override predicate getEndpoints(DataFlow::Node n) {
     exists(DataFlow::CallNode call | n = call.getAnArgument() |
       call instanceof StringOps::StartsWith
     )
@@ -439,7 +439,7 @@ private class StringEndsWithCharacteristic extends NotASinkCharacteristic,
   OtherModeledArgumentCharacteristic {
   StringEndsWithCharacteristic() { this = "StringEndsWith" }
 
-  override predicate appliesToEndpoint(DataFlow::Node n) {
+  override predicate getEndpoints(DataFlow::Node n) {
     exists(DataFlow::CallNode call | n = call.getAnArgument() | call instanceof StringOps::EndsWith)
   }
 }
@@ -448,7 +448,7 @@ private class StringRegExpTestCharacteristic extends NotASinkCharacteristic,
   OtherModeledArgumentCharacteristic {
   StringRegExpTestCharacteristic() { this = "StringRegExpTest" }
 
-  override predicate appliesToEndpoint(DataFlow::Node n) {
+  override predicate getEndpoints(DataFlow::Node n) {
     exists(DataFlow::CallNode call | n = call.getAnArgument() |
       call instanceof StringOps::RegExpTest
     )
@@ -459,7 +459,7 @@ private class EventRegistrationCharacteristic extends NotASinkCharacteristic,
   OtherModeledArgumentCharacteristic {
   EventRegistrationCharacteristic() { this = "EventRegistration" }
 
-  override predicate appliesToEndpoint(DataFlow::Node n) {
+  override predicate getEndpoints(DataFlow::Node n) {
     exists(DataFlow::CallNode call | n = call.getAnArgument() | call instanceof EventRegistration)
   }
 }
@@ -468,7 +468,7 @@ private class EventDispatchCharacteristic extends NotASinkCharacteristic,
   OtherModeledArgumentCharacteristic {
   EventDispatchCharacteristic() { this = "EventDispatch" }
 
-  override predicate appliesToEndpoint(DataFlow::Node n) {
+  override predicate getEndpoints(DataFlow::Node n) {
     exists(DataFlow::CallNode call | n = call.getAnArgument() | call instanceof EventDispatch)
   }
 }
@@ -477,7 +477,7 @@ private class MembershipCandidateTestCharacteristic extends NotASinkCharacterist
   OtherModeledArgumentCharacteristic {
   MembershipCandidateTestCharacteristic() { this = "MembershipCandidateTest" }
 
-  override predicate appliesToEndpoint(DataFlow::Node n) {
+  override predicate getEndpoints(DataFlow::Node n) {
     exists(DataFlow::CallNode call | n = call.getAnArgument() |
       call = any(MembershipCandidate c).getTest()
     )
@@ -488,7 +488,7 @@ private class FileSystemAccessCharacteristic extends NotASinkCharacteristic,
   OtherModeledArgumentCharacteristic {
   FileSystemAccessCharacteristic() { this = "FileSystemAccess" }
 
-  override predicate appliesToEndpoint(DataFlow::Node n) {
+  override predicate getEndpoints(DataFlow::Node n) {
     exists(DataFlow::CallNode call | n = call.getAnArgument() | call instanceof FileSystemAccess)
   }
 }
@@ -497,7 +497,7 @@ private class DatabaseAccessCharacteristic extends NotASinkCharacteristic,
   OtherModeledArgumentCharacteristic {
   DatabaseAccessCharacteristic() { this = "DatabaseAccess" }
 
-  override predicate appliesToEndpoint(DataFlow::Node n) {
+  override predicate getEndpoints(DataFlow::Node n) {
     // TODO database accesses are less well defined than database query sinks, so this may cover unmodeled sinks on
     // existing database models
     exists(DataFlow::CallNode call | n = call.getAnArgument() |
@@ -512,7 +512,7 @@ private class DatabaseAccessCharacteristic extends NotASinkCharacteristic,
 private class DomCharacteristic extends NotASinkCharacteristic, OtherModeledArgumentCharacteristic {
   DomCharacteristic() { this = "DOM" }
 
-  override predicate appliesToEndpoint(DataFlow::Node n) {
+  override predicate getEndpoints(DataFlow::Node n) {
     exists(DataFlow::CallNode call | n = call.getAnArgument() | call = DOM::domValueRef())
   }
 }
@@ -521,7 +521,7 @@ private class NextFunctionCallCharacteristic extends NotASinkCharacteristic,
   OtherModeledArgumentCharacteristic {
   NextFunctionCallCharacteristic() { this = "NextFunctionCall" }
 
-  override predicate appliesToEndpoint(DataFlow::Node n) {
+  override predicate getEndpoints(DataFlow::Node n) {
     exists(DataFlow::CallNode call | n = call.getAnArgument() |
       call.getCalleeName() = "next" and
       exists(DataFlow::FunctionNode f | call = f.getLastParameter().getACall())
@@ -533,7 +533,7 @@ private class DojoRequireCharacteristic extends NotASinkCharacteristic,
   OtherModeledArgumentCharacteristic {
   DojoRequireCharacteristic() { this = "DojoRequire" }
 
-  override predicate appliesToEndpoint(DataFlow::Node n) {
+  override predicate getEndpoints(DataFlow::Node n) {
     exists(DataFlow::CallNode call | n = call.getAnArgument() |
       call = DataFlow::globalVarRef("dojo").getAPropertyRead("require").getACall()
     )
@@ -544,7 +544,7 @@ private class Base64ManipulationCharacteristic extends NotASinkCharacteristic,
   OtherModeledArgumentCharacteristic {
   Base64ManipulationCharacteristic() { this = "Base64Manipulation" }
 
-  override predicate appliesToEndpoint(DataFlow::Node n) {
+  override predicate getEndpoints(DataFlow::Node n) {
     exists(Base64::Decode d | n = d.getInput()) or
     exists(Base64::Encode d | n = d.getInput())
   }
@@ -554,7 +554,7 @@ private class ArgumentToArrayCharacteristic extends ArgumentToBuiltinFunctionCha
   LikelyNotASinkCharacteristic {
   ArgumentToArrayCharacteristic() { this = "ArgumentToArray" }
 
-  override predicate appliesToEndpoint(DataFlow::Node n) {
+  override predicate getEndpoints(DataFlow::Node n) {
     exists(DataFlow::SourceNode builtin, DataFlow::SourceNode receiver, DataFlow::InvokeNode invk |
       builtin instanceof DataFlow::ArrayCreationNode
     |
@@ -569,7 +569,7 @@ private class ArgumentToBuiltinGlobalVarRefCharacteristic extends ArgumentToBuil
   LikelyNotASinkCharacteristic {
   ArgumentToBuiltinGlobalVarRefCharacteristic() { this = "ArgumentToBuiltinGlobalVarRef" }
 
-  override predicate appliesToEndpoint(DataFlow::Node n) {
+  override predicate getEndpoints(DataFlow::Node n) {
     exists(DataFlow::SourceNode builtin, DataFlow::SourceNode receiver, DataFlow::InvokeNode invk |
       builtin =
         DataFlow::globalVarRef([
@@ -588,7 +588,7 @@ private class ConstantReceiverCharacteristic extends ArgumentToBuiltinFunctionCh
   NotASinkCharacteristic {
   ConstantReceiverCharacteristic() { this = "ConstantReceiver" }
 
-  override predicate appliesToEndpoint(DataFlow::Node n) {
+  override predicate getEndpoints(DataFlow::Node n) {
     exists(Expr primitive, MethodCallExpr c |
       primitive instanceof ConstantString or
       primitive instanceof NumberLiteral or
@@ -604,7 +604,7 @@ private class BuiltinCallNameCharacteristic extends ArgumentToBuiltinFunctionCha
   NotASinkCharacteristic {
   BuiltinCallNameCharacteristic() { this = "BuiltinCallName" }
 
-  override predicate appliesToEndpoint(DataFlow::Node n) {
+  override predicate getEndpoints(DataFlow::Node n) {
     exists(DataFlow::CallNode call |
       call.getAnArgument() = n and
       call.getCalleeName() =
@@ -633,7 +633,7 @@ abstract private class StandardEndpointFilterCharacteristic extends EndpointFilt
   bindingset[this]
   StandardEndpointFilterCharacteristic() { any() }
 
-  override predicate hasImplications(
+  override predicate getImplications(
     EndpointType endpointClass, boolean isPositiveIndicator, float confidence
   ) {
     endpointClass instanceof NegativeType and
@@ -645,7 +645,7 @@ abstract private class StandardEndpointFilterCharacteristic extends EndpointFilt
 class IsArgumentToModeledFunctionCharacteristic extends StandardEndpointFilterCharacteristic {
   IsArgumentToModeledFunctionCharacteristic() { this = "argument to modeled function" }
 
-  override predicate appliesToEndpoint(DataFlow::Node n) {
+  override predicate getEndpoints(DataFlow::Node n) {
     exists(DataFlow::InvokeNode invk, DataFlow::Node known |
       invk.getAnArgument() = n and
       invk.getAnArgument() = known and
@@ -655,7 +655,7 @@ class IsArgumentToModeledFunctionCharacteristic extends StandardEndpointFilterCh
         isKnownStepSrc(known)
         or
         exists(OtherModeledArgumentCharacteristic characteristic |
-          characteristic.appliesToEndpoint(known)
+          characteristic.getEndpoints(known)
         )
       )
     )
@@ -665,7 +665,7 @@ class IsArgumentToModeledFunctionCharacteristic extends StandardEndpointFilterCh
 private class IsArgumentToSinklessLibraryCharacteristic extends StandardEndpointFilterCharacteristic {
   IsArgumentToSinklessLibraryCharacteristic() { this = "argument to sinkless library" }
 
-  override predicate appliesToEndpoint(DataFlow::Node n) {
+  override predicate getEndpoints(DataFlow::Node n) {
     exists(DataFlow::InvokeNode invk, DataFlow::SourceNode commonSafeLibrary, string libraryName |
       libraryName = ["slugify", "striptags", "marked"]
     |
@@ -679,7 +679,7 @@ private class IsArgumentToSinklessLibraryCharacteristic extends StandardEndpoint
 private class IsSanitizerCharacteristic extends StandardEndpointFilterCharacteristic {
   IsSanitizerCharacteristic() { this = "sanitizer" }
 
-  override predicate appliesToEndpoint(DataFlow::Node n) {
+  override predicate getEndpoints(DataFlow::Node n) {
     exists(DataFlow::CallNode call | n = call.getAnArgument() |
       call.getCalleeName().regexpMatch("(?i).*(escape|valid(ate)?|sanitize|purify).*")
     )
@@ -689,7 +689,7 @@ private class IsSanitizerCharacteristic extends StandardEndpointFilterCharacteri
 private class IsPredicateCharacteristic extends StandardEndpointFilterCharacteristic {
   IsPredicateCharacteristic() { this = "predicate" }
 
-  override predicate appliesToEndpoint(DataFlow::Node n) {
+  override predicate getEndpoints(DataFlow::Node n) {
     exists(DataFlow::CallNode call | n = call.getAnArgument() |
       call.getCalleeName().regexpMatch("(equals|(|is|has|can)(_|[A-Z])).*")
     )
@@ -699,7 +699,7 @@ private class IsPredicateCharacteristic extends StandardEndpointFilterCharacteri
 private class IsHashCharacteristic extends StandardEndpointFilterCharacteristic {
   IsHashCharacteristic() { this = "hash" }
 
-  override predicate appliesToEndpoint(DataFlow::Node n) {
+  override predicate getEndpoints(DataFlow::Node n) {
     exists(DataFlow::CallNode call | n = call.getAnArgument() |
       call.getCalleeName().regexpMatch("(?i)^(sha\\d*|md5|hash)$")
     )
@@ -709,7 +709,7 @@ private class IsHashCharacteristic extends StandardEndpointFilterCharacteristic
 private class IsNumericCharacteristic extends StandardEndpointFilterCharacteristic {
   IsNumericCharacteristic() { this = "numeric" }
 
-  override predicate appliesToEndpoint(DataFlow::Node n) {
+  override predicate getEndpoints(DataFlow::Node n) {
     SyntacticHeuristics::isReadFrom(n, ".*index.*")
   }
 }
@@ -721,7 +721,7 @@ private class InIrrelevantFileCharacteristic extends StandardEndpointFilterChara
     this = "in " + category + " file" and category = ["externs", "generated", "library", "test"]
   }
 
-  override predicate appliesToEndpoint(DataFlow::Node n) {
+  override predicate getEndpoints(DataFlow::Node n) {
     // Ignore candidate sinks within externs, generated, library, and test code
     ClassifyFiles::classify(n.getFile(), category)
   }
@@ -732,7 +732,7 @@ abstract private class NosqlInjectionSinkEndpointFilterCharacteristic extends En
   bindingset[this]
   NosqlInjectionSinkEndpointFilterCharacteristic() { any() }
 
-  override predicate hasImplications(
+  override predicate getImplications(
     EndpointType endpointClass, boolean isPositiveIndicator, float confidence
   ) {
     endpointClass instanceof NosqlInjectionSinkType and
@@ -744,7 +744,7 @@ abstract private class NosqlInjectionSinkEndpointFilterCharacteristic extends En
 private class DatabaseAccessCallHeuristicCharacteristic extends NosqlInjectionSinkEndpointFilterCharacteristic {
   DatabaseAccessCallHeuristicCharacteristic() { this = "matches database access call heuristic" }
 
-  override predicate appliesToEndpoint(DataFlow::Node n) {
+  override predicate getEndpoints(DataFlow::Node n) {
     exists(DataFlow::MethodCallNode call | n = call.getAnArgument() |
       // additional databases accesses that aren't modeled yet
       call.getMethodName() = ["create", "createCollection", "createIndexes"]
@@ -764,7 +764,7 @@ private class ModeledSinkCharacteristic extends NosqlInjectionSinkEndpointFilter
     )
   }
 
-  override predicate appliesToEndpoint(DataFlow::Node n) {
+  override predicate getEndpoints(DataFlow::Node n) {
     exists(DataFlow::CallNode call | n = call.getAnArgument() |
       // Remove modeled sinks
       isArgumentToKnownLibrarySinkFunction(n)
@@ -775,7 +775,7 @@ private class ModeledSinkCharacteristic extends NosqlInjectionSinkEndpointFilter
 private class PredecessorInModeledFlowStepCharacteristic extends NosqlInjectionSinkEndpointFilterCharacteristic {
   PredecessorInModeledFlowStepCharacteristic() { this = "predecessor in a modeled flow step" }
 
-  override predicate appliesToEndpoint(DataFlow::Node n) {
+  override predicate getEndpoints(DataFlow::Node n) {
     exists(DataFlow::CallNode call | n = call.getAnArgument() |
       // Remove common kinds of unlikely sinks
       isKnownStepSrc(n)
@@ -786,7 +786,7 @@ private class PredecessorInModeledFlowStepCharacteristic extends NosqlInjectionS
 private class ModeledDatabaseAccessCharacteristic extends NosqlInjectionSinkEndpointFilterCharacteristic {
   ModeledDatabaseAccessCharacteristic() { this = "modeled database access" }
 
-  override predicate appliesToEndpoint(DataFlow::Node n) {
+  override predicate getEndpoints(DataFlow::Node n) {
     exists(DataFlow::CallNode call | n = call.getAnArgument() |
       // Remove modeled database calls. Arguments to modeled calls are very likely to be modeled
       // as sinks if they are true positives. Therefore arguments that are not modeled as sinks
@@ -799,7 +799,7 @@ private class ModeledDatabaseAccessCharacteristic extends NosqlInjectionSinkEndp
 private class ReceiverIsHttpRequestExpressionCharacteristic extends NosqlInjectionSinkEndpointFilterCharacteristic {
   ReceiverIsHttpRequestExpressionCharacteristic() { this = "receiver is a HTTP request expression" }
 
-  override predicate appliesToEndpoint(DataFlow::Node n) {
+  override predicate getEndpoints(DataFlow::Node n) {
     exists(DataFlow::CallNode call | n = call.getAnArgument() |
       // Remove calls to APIs that aren't relevant to NoSQL injection
       call.getReceiver() instanceof Http::RequestNode
@@ -812,7 +812,7 @@ private class ReceiverIsHttpResponseExpressionCharacteristic extends NosqlInject
     this = "receiver is a HTTP response expression"
   }
 
-  override predicate appliesToEndpoint(DataFlow::Node n) {
+  override predicate getEndpoints(DataFlow::Node n) {
     exists(DataFlow::CallNode call | n = call.getAnArgument() |
       // Remove calls to APIs that aren't relevant to NoSQL injection
       call.getReceiver() instanceof Http::ResponseNode
@@ -825,7 +825,7 @@ private class NotDirectArgumentToLikelyExternalLibraryCallOrHeuristicSinkNosqlCh
     this = "not a direct argument to a likely external library call or a heuristic sink (nosql)"
   }
 
-  override predicate appliesToEndpoint(DataFlow::Node n) {
+  override predicate getEndpoints(DataFlow::Node n) {
     // Require NoSQL injection sink candidates to be (a) direct arguments to external library calls
     // or (b) heuristic sinks for NoSQL injection.
     //
@@ -867,7 +867,7 @@ abstract private class SqlInjectionSinkEndpointFilterCharacteristic extends Endp
   bindingset[this]
   SqlInjectionSinkEndpointFilterCharacteristic() { any() }
 
-  override predicate hasImplications(
+  override predicate getImplications(
     EndpointType endpointClass, boolean isPositiveIndicator, float confidence
   ) {
     endpointClass instanceof SqlInjectionSinkType and
@@ -879,7 +879,7 @@ abstract private class SqlInjectionSinkEndpointFilterCharacteristic extends Endp
 private class PreparedSqlStatementCharacteristic extends SqlInjectionSinkEndpointFilterCharacteristic {
   PreparedSqlStatementCharacteristic() { this = "prepared SQL statement" }
 
-  override predicate appliesToEndpoint(DataFlow::Node n) {
+  override predicate getEndpoints(DataFlow::Node n) {
     exists(DataFlow::CallNode call | n = call.getAnArgument() |
       // prepared statements for SQL
       any(DataFlow::CallNode cn | cn.getCalleeName() = "prepare")
@@ -892,7 +892,7 @@ private class PreparedSqlStatementCharacteristic extends SqlInjectionSinkEndpoin
 private class ArrayCreationCharacteristic extends SqlInjectionSinkEndpointFilterCharacteristic {
   ArrayCreationCharacteristic() { this = "array creation" }
 
-  override predicate appliesToEndpoint(DataFlow::Node n) {
+  override predicate getEndpoints(DataFlow::Node n) {
     exists(DataFlow::CallNode call | n = call.getAnArgument() |
       n instanceof DataFlow::ArrayCreationNode
     )
@@ -902,7 +902,7 @@ private class ArrayCreationCharacteristic extends SqlInjectionSinkEndpointFilter
 private class HtmlOrRenderingCharacteristic extends SqlInjectionSinkEndpointFilterCharacteristic {
   HtmlOrRenderingCharacteristic() { this = "HTML / rendering" }
 
-  override predicate appliesToEndpoint(DataFlow::Node n) {
+  override predicate getEndpoints(DataFlow::Node n) {
     exists(DataFlow::CallNode call | n = call.getAnArgument() |
       // UI is unrelated to SQL
       call.getCalleeName().regexpMatch("(?i).*(render|html).*")
@@ -915,7 +915,7 @@ private class NotAnArgumentToLikelyExternalLibraryCallOrHeuristicSinkCharacteris
     this = "not an argument to a likely external library call or a heuristic sink"
   }
 
-  override predicate appliesToEndpoint(DataFlow::Node n) {
+  override predicate getEndpoints(DataFlow::Node n) {
     // Require SQL injection sink candidates to be (a) arguments to external library calls
     // (possibly indirectly), or (b) heuristic sinks.
     //
@@ -938,7 +938,7 @@ abstract private class TaintedPathSinkEndpointFilterCharacteristic extends Endpo
   bindingset[this]
   TaintedPathSinkEndpointFilterCharacteristic() { any() }
 
-  override predicate hasImplications(
+  override predicate getImplications(
     EndpointType endpointClass, boolean isPositiveIndicator, float confidence
   ) {
     endpointClass instanceof TaintedPathSinkType and
@@ -953,7 +953,7 @@ private class NotDirectArgumentToLikelyExternalLibraryCallOrHeuristicSinkTainted
       "not a direct argument to a likely external library call or a heuristic sink (tainted path)"
   }
 
-  override predicate appliesToEndpoint(DataFlow::Node n) {
+  override predicate getEndpoints(DataFlow::Node n) {
     // Require path injection sink candidates to be (a) arguments to external library calls
     // (possibly indirectly), or (b) heuristic sinks.
     //
@@ -990,7 +990,7 @@ abstract private class XssSinkEndpointFilterCharacteristic extends EndpointFilte
   bindingset[this]
   XssSinkEndpointFilterCharacteristic() { any() }
 
-  override predicate hasImplications(
+  override predicate getImplications(
     EndpointType endpointClass, boolean isPositiveIndicator, float confidence
   ) {
     endpointClass instanceof XssSinkType and
@@ -1004,7 +1004,7 @@ private class SetStateCallsInReactApplicationsCharacteristic extends XssSinkEndp
     this = "setState calls ought to be safe in react applications"
   }
 
-  override predicate appliesToEndpoint(DataFlow::Node n) {
+  override predicate getEndpoints(DataFlow::Node n) {
     exists(DataFlow::CallNode call | n = call.getAnArgument() | call.getCalleeName() = "setState")
   }
 }
@@ -1014,7 +1014,7 @@ private class NotDirectArgumentToLikelyExternalLibraryCallOrHeuristicSinkXssChar
     this = "not a direct argument to a likely external library call or a heuristic sink (xss)"
   }
 
-  override predicate appliesToEndpoint(DataFlow::Node n) {
+  override predicate getEndpoints(DataFlow::Node n) {
     // Require XSS sink candidates to be (a) arguments to external library calls (possibly
     // indirectly), or (b) heuristic sinks.
     //

javascript/ql/experimental/adaptivethreatmodeling/lib/experimental/adaptivethreatmodeling/XssThroughDomATM.qll

@@ -1,88 +0,0 @@
-/**
- * For internal use only.
- *
- * A taint-tracking configuration for reasoning about XSS through the DOM.
- * Defines shared code used by the XSS Through DOM boosted query.
- */
-
-private import semmle.javascript.heuristics.SyntacticHeuristics
-private import semmle.javascript.security.dataflow.DomBasedXssCustomizations
-private import semmle.javascript.dataflow.InferredTypes
-private import semmle.javascript.security.dataflow.XssThroughDomCustomizations::XssThroughDom as XssThroughDom
-private import semmle.javascript.security.dataflow.UnsafeJQueryPluginCustomizations::UnsafeJQueryPlugin as UnsafeJQuery
-import AdaptiveThreatModeling
-
-class XssThroughDomAtmConfig extends AtmConfig {
-  XssThroughDomAtmConfig() { this = "XssThroughDomAtmConfig" }
-
-  override predicate isKnownSource(DataFlow::Node source) {
-    source instanceof XssThroughDom::Source
-  }
-
-  override EndpointType getASinkEndpointType() { result instanceof XssSinkType }
-
-  override predicate isSanitizer(DataFlow::Node node) {
-    super.isSanitizer(node) or
-    node instanceof DomBasedXss::Sanitizer
-  }
-
-  override predicate isSanitizerGuard(TaintTracking::SanitizerGuardNode guard) {
-    guard instanceof TypeTestGuard or
-    guard instanceof UnsafeJQuery::PropertyPresenceSanitizer or
-    guard instanceof UnsafeJQuery::NumberGuard or
-    guard instanceof PrefixStringSanitizer or
-    guard instanceof QuoteGuard or
-    guard instanceof ContainsHtmlGuard
-  }
-
-  override predicate isSanitizerEdge(DataFlow::Node pred, DataFlow::Node succ) {
-    DomBasedXss::isOptionallySanitizedEdge(pred, succ)
-  }
-}
-
-/**
- * A test of form `typeof x === "something"`, preventing `x` from being a string in some cases.
- *
- * This sanitizer helps prune infeasible paths in type-overloaded functions.
- */
-class TypeTestGuard extends TaintTracking::SanitizerGuardNode, DataFlow::ValueNode {
-  override EqualityTest astNode;
-  Expr operand;
-  boolean polarity;
-
-  TypeTestGuard() {
-    exists(TypeofTag tag | TaintTracking::isTypeofGuard(astNode, operand, tag) |
-      // typeof x === "string" sanitizes `x` when it evaluates to false
-      tag = "string" and
-      polarity = astNode.getPolarity().booleanNot()
-      or
-      // typeof x === "object" sanitizes `x` when it evaluates to true
-      tag != "string" and
-      polarity = astNode.getPolarity()
-    )
-  }
-
-  override predicate sanitizes(boolean outcome, Expr e) {
-    polarity = outcome and
-    e = operand
-  }
-}
-
-private import semmle.javascript.security.dataflow.Xss::Shared as Shared
-
-private class PrefixStringSanitizer extends TaintTracking::SanitizerGuardNode,
-  DomBasedXss::PrefixStringSanitizer {
-  PrefixStringSanitizer() { this = this }
-}
-
-private class PrefixString extends DataFlow::FlowLabel, DomBasedXss::PrefixString {
-  PrefixString() { this = this }
-}
-
-private class QuoteGuard extends TaintTracking::SanitizerGuardNode, Shared::QuoteGuard {
-  QuoteGuard() { this = this }
-}
-
-private class ContainsHtmlGuard extends TaintTracking::SanitizerGuardNode, Shared::ContainsHtmlGuard {
-  ContainsHtmlGuard() { this = this }
-}

javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/DebugResultInclusion.ql

@@ -16,7 +16,6 @@ private import experimental.adaptivethreatmodeling.NosqlInjectionATM as NosqlInj
 private import experimental.adaptivethreatmodeling.SqlInjectionATM as SqlInjectionAtm
 private import experimental.adaptivethreatmodeling.TaintedPathATM as TaintedPathAtm
 private import experimental.adaptivethreatmodeling.XssATM as XssAtm
-private import experimental.adaptivethreatmodeling.XssThroughDomATM as XssThroughDomAtm
 
 string getAReasonSinkExcluded(DataFlow::Node sinkCandidate, Query query) {
   query instanceof NosqlInjectionQuery and
@@ -30,9 +29,6 @@ string getAReasonSinkExcluded(DataFlow::Node sinkCandidate, Query query) {
   or
   query instanceof XssQuery and
   result = any(XssAtm::DomBasedXssAtmConfig cfg).getAReasonSinkExcluded(sinkCandidate)
-  or
-  query instanceof XssThroughDomQuery and
-  result = any(XssThroughDomAtm::XssThroughDomAtmConfig cfg).getAReasonSinkExcluded(sinkCandidate)
 }
 
 pragma[inline]

javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/extraction/ExtractEndpointDataTraining.qll

@@ -14,7 +14,6 @@ private import experimental.adaptivethreatmodeling.NosqlInjectionATM as NosqlInj
 private import experimental.adaptivethreatmodeling.SqlInjectionATM as SqlInjectionAtm
 private import experimental.adaptivethreatmodeling.TaintedPathATM as TaintedPathAtm
 private import experimental.adaptivethreatmodeling.XssATM as XssAtm
-private import experimental.adaptivethreatmodeling.XssThroughDomATM as XssThroughDomAtm
 
 /**
  * Gets the set of featureName-featureValue pairs for each endpoint in the training set.
@@ -54,7 +53,7 @@ predicate tokenFeatures(DataFlow::Node endpoint, string featureName, string feat
 query predicate trainingEndpoints(
   DataFlow::Node endpoint, EndpointType endpointClass, EndpointCharacteristic characteristic
 ) {
-  characteristic.appliesToEndpoint(endpoint) and
+  characteristic.getEndpoints(endpoint) and
   // Only consider the source code for the project being analyzed.
   exists(endpoint.getFile().getRelativePath()) and
   // Only select endpoints that can be part of a tainted flow: Constant expressions always evaluate to a constant
@@ -70,7 +69,7 @@ query predicate trainingEndpoints(
   not (
     endpointClass instanceof NegativeType and
     exists(EndpointCharacteristic c |
-      c.appliesToEndpoint(endpoint) and
+      c.getEndpoints(endpoint) and
       c instanceof LikelyNotASinkCharacteristic
     )
   ) and
@@ -82,8 +81,8 @@ query predicate trainingEndpoints(
     // If the list of characteristics includes positive indicators with high confidence for this class, select this as a
     // training sample belonging to the class.
     exists(EndpointCharacteristic characteristic2, float confidence |
-      characteristic2.appliesToEndpoint(endpoint) and
-      characteristic2.hasImplications(endpointClass, true, confidence) and
+      characteristic2.getEndpoints(endpoint) and
+      characteristic2.getImplications(endpointClass, true, confidence) and
       confidence >= characteristic2.getHighConfidenceThreshold()
     ) and
     (
@@ -94,8 +93,8 @@ query predicate trainingEndpoints(
       not endpointClass instanceof NegativeType
       or
       not exists(EndpointCharacteristic characteristic3, float confidence3, EndpointType posClass |
-        characteristic3.appliesToEndpoint(endpoint) and
-        characteristic3.hasImplications(posClass, true, confidence3) and
+        characteristic3.getEndpoints(endpoint) and
+        characteristic3.getImplications(posClass, true, confidence3) and
         confidence3 >= characteristic3.getHighConfidenceThreshold() and
         not posClass instanceof NegativeType
       )
@@ -107,8 +106,8 @@ query predicate trainingEndpoints(
     endpointClass instanceof NegativeType and
     forall(EndpointType otherClass | not otherClass instanceof NegativeType |
       exists(EndpointCharacteristic characteristic2, float confidence |
-        characteristic2.appliesToEndpoint(endpoint) and
-        characteristic2.hasImplications(otherClass, false, confidence) and
+        characteristic2.getEndpoints(endpoint) and
+        characteristic2.getImplications(otherClass, false, confidence) and
         confidence >= characteristic2.getHighConfidenceThreshold()
       )
     )
@@ -181,15 +180,15 @@ query predicate reformattedTrainingEndpoints(
       // The reason, or reasons, why the endpoint was labeled NotASink for this query, only for negative examples.
       key = "notASinkReason" and
       exists(EndpointCharacteristic characteristic, EndpointType endpointClass |
-        characteristic.appliesToEndpoint(endpoint) and
-        characteristic.hasImplications(endpointClass, true, _) and
+        characteristic.getEndpoints(endpoint) and
+        characteristic.getImplications(endpointClass, true, _) and
         endpointClass instanceof NegativeType and
         value = characteristic
       ) and
       // Don't include a notASinkReason for endpoints that are also known sinks.
       not exists(EndpointCharacteristic characteristic3, float confidence3, EndpointType posClass |
-        characteristic3.appliesToEndpoint(endpoint) and
-        characteristic3.hasImplications(posClass, true, confidence3) and
+        characteristic3.getEndpoints(endpoint) and
+        characteristic3.getImplications(posClass, true, confidence3) and
         confidence3 >= characteristic3.getHighConfidenceThreshold() and
         not posClass instanceof NegativeType
       ) and
@@ -215,8 +214,6 @@ DataFlow::Configuration getDataFlowCfg(Query query) {
   query instanceof TaintedPathQuery and result instanceof TaintedPathAtm::TaintedPathAtmConfig
   or
   query instanceof XssQuery and result instanceof XssAtm::DomBasedXssAtmConfig
-  or
-  query instanceof XssThroughDomQuery and result instanceof XssThroughDomAtm::XssThroughDomAtmConfig
 }
 
 // TODO: Delete this once we are no longer surfacing `hasFlowFromSource`.

javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/extraction/ExtractEndpointMapping.ql

@@ -8,7 +8,6 @@ import experimental.adaptivethreatmodeling.SqlInjectionATM as SqlInjectionAtm
 import experimental.adaptivethreatmodeling.NosqlInjectionATM as NosqlInjectionAtm
 import experimental.adaptivethreatmodeling.TaintedPathATM as TaintedPathAtm
 import experimental.adaptivethreatmodeling.XssATM as XssAtm
-import experimental.adaptivethreatmodeling.XssThroughDomATM as XssThroughDomAtm
 import experimental.adaptivethreatmodeling.AdaptiveThreatModeling
 
 from string queryName, AtmConfig c, EndpointType e
@@ -24,8 +23,6 @@ where
     c instanceof TaintedPathAtm::TaintedPathAtmConfig
     or
     queryName = "Xss" and c instanceof XssAtm::DomBasedXssAtmConfig
-    or
-    queryName = "XssThroughDom" and c instanceof XssThroughDomAtm::XssThroughDomAtmConfig
   ) and
   e = c.getASinkEndpointType()
 select queryName, e.getEncoding() as label

javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/extraction/Queries.qll

@@ -8,8 +8,7 @@ newtype TQuery =
   TNosqlInjectionQuery() or
   TSqlInjectionQuery() or
   TTaintedPathQuery() or
-  TXssQuery() or
-  TXssThroughDomQuery()
+  TXssQuery()
 
 abstract class Query extends TQuery {
   abstract string getName();
@@ -32,7 +31,3 @@ class TaintedPathQuery extends Query, TTaintedPathQuery {
 class XssQuery extends Query, TXssQuery {
   override string getName() { result = "Xss" }
 }
-
-class XssThroughDomQuery extends Query, TXssThroughDomQuery {
-  override string getName() { result = "XssThroughDom" }
-}

javascript/ql/experimental/adaptivethreatmodeling/src/NosqlInjectionATM.ql

@@ -18,7 +18,7 @@ import DataFlow::PathGraph
 import experimental.adaptivethreatmodeling.NosqlInjectionATM
 
 from AtmConfig cfg, DataFlow::PathNode source, DataFlow::PathNode sink, float score
-where cfg.hasBoostedFlowPath(source, sink, score)
+where cfg.getAlerts(source, sink, score)
 select sink.getNode(), source, sink,
   "(Experimental) This may be a database query that depends on $@. Identified using machine learning.",
   source.getNode(), "a user-provided value", score

javascript/ql/experimental/adaptivethreatmodeling/src/SqlInjectionATM.ql

@@ -18,7 +18,7 @@ import ATM::ResultsInfo
 import DataFlow::PathGraph
 
 from AtmConfig cfg, DataFlow::PathNode source, DataFlow::PathNode sink, float score
-where cfg.hasBoostedFlowPath(source, sink, score)
+where cfg.getAlerts(source, sink, score)
 select sink.getNode(), source, sink,
   "(Experimental) This may be a database query that depends on $@. Identified using machine learning.",
   source.getNode(), "a user-provided value", score

javascript/ql/experimental/adaptivethreatmodeling/src/TaintedPathATM.ql

@@ -22,7 +22,7 @@ import DataFlow::PathGraph
 import experimental.adaptivethreatmodeling.TaintedPathATM
 
 from AtmConfig cfg, DataFlow::PathNode source, DataFlow::PathNode sink, float score
-where cfg.hasBoostedFlowPath(source, sink, score)
+where cfg.getAlerts(source, sink, score)
 select sink.getNode(), source, sink,
   "(Experimental) This may be a path that depends on $@. Identified using machine learning.",
   source.getNode(), "a user-provided value", score

javascript/ql/experimental/adaptivethreatmodeling/src/XssATM.ql

@@ -19,7 +19,7 @@ import DataFlow::PathGraph
 import experimental.adaptivethreatmodeling.XssATM
 
 from AtmConfig cfg, DataFlow::PathNode source, DataFlow::PathNode sink, float score
-where cfg.hasBoostedFlowPath(source, sink, score)
+where cfg.getAlerts(source, sink, score)
 select sink.getNode(), source, sink,
   "(Experimental) This may be a cross-site scripting vulnerability due to $@. Identified using machine learning.",
   source.getNode(), "a user-provided value", score

javascript/ql/experimental/adaptivethreatmodeling/src/XssThroughDomATM.ql

@@ -1,25 +0,0 @@
-/**
- * For internal use only.
- *
- * @name DOM text reinterpreted as HTML (experimental)
- * @description Reinterpreting text from the DOM as HTML can lead
- *              to a cross-site scripting vulnerability.
- * @kind path-problem
- * @scored
- * @problem.severity error
- * @security-severity 6.1
- * @id js/ml-powered/xss-through-dom
- * @tags experimental security
- *       external/cwe/cwe-079 external/cwe/cwe-116
- */
-
-import javascript
-import ATM::ResultsInfo
-import DataFlow::PathGraph
-import experimental.adaptivethreatmodeling.XssThroughDomATM
-
-from AtmConfig cfg, DataFlow::PathNode source, DataFlow::PathNode sink, float score
-where cfg.hasBoostedFlowPath(source, sink, score)
-select sink.getNode(), source, sink,
-  "(Experimental) $@ may be reinterpreted as HTML without escaping meta-characters. Identified using machine learning.",
-  source.getNode(), "DOM text", score

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/EndpointFeatures.ql

@@ -11,7 +11,6 @@ import experimental.adaptivethreatmodeling.NosqlInjectionATM as NosqlInjectionAt
 import experimental.adaptivethreatmodeling.SqlInjectionATM as SqlInjectionAtm
 import experimental.adaptivethreatmodeling.TaintedPathATM as TaintedPathAtm
 import experimental.adaptivethreatmodeling.XssATM as XssAtm
-import experimental.adaptivethreatmodeling.XssThroughDomATM as XssThroughDomAtm
 import experimental.adaptivethreatmodeling.EndpointFeatures as EndpointFeatures
 import extraction.NoFeaturizationRestrictionsConfig
 private import experimental.adaptivethreatmodeling.EndpointCharacteristics as EndpointCharacteristics
@@ -22,9 +21,8 @@ query predicate tokenFeatures(DataFlow::Node endpoint, string featureName, strin
     not exists(any(SqlInjectionAtm::SqlInjectionAtmConfig cfg).getAReasonSinkExcluded(endpoint)) or
     not exists(any(TaintedPathAtm::TaintedPathAtmConfig cfg).getAReasonSinkExcluded(endpoint)) or
     not exists(any(XssAtm::DomBasedXssAtmConfig cfg).getAReasonSinkExcluded(endpoint)) or
-    not exists(any(XssThroughDomAtm::XssThroughDomAtmConfig cfg).getAReasonSinkExcluded(endpoint)) or
     any(EndpointCharacteristics::IsArgumentToModeledFunctionCharacteristic characteristic)
-        .appliesToEndpoint(endpoint)
+        .getEndpoints(endpoint)
   ) and
   EndpointFeatures::tokenFeatures(endpoint, featureName, featureValue)
 }

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/FilteredTruePositives.expected

@@ -14,11 +14,3 @@ xssFilteredTruePositives
 | autogenerated/Xss/DomBasedXss/tst.js:316:35:316:42 | location | not a direct argument to a likely external library call or a heuristic sink (xss) |
 | autogenerated/Xss/DomBasedXss/typeahead.js:10:16:10:18 | loc | not a direct argument to a likely external library call or a heuristic sink (xss) |
 | autogenerated/Xss/DomBasedXss/typeahead.js:25:18:25:20 | val | not a direct argument to a likely external library call or a heuristic sink (xss) |
-xssThroughDomFilteredTruePositives
-| autogenerated/Xss/DomBasedXss/d3.js:12:20:12:29 | getTaint() | not a direct argument to a likely external library call or a heuristic sink (xss) |
-| autogenerated/Xss/DomBasedXss/d3.js:14:20:14:29 | getTaint() | not a direct argument to a likely external library call or a heuristic sink (xss) |
-| autogenerated/Xss/DomBasedXss/express.js:7:15:7:33 | req.param("wobble") | not a direct argument to a likely external library call or a heuristic sink (xss) |
-| autogenerated/Xss/DomBasedXss/jwt-server.js:11:19:11:29 | decoded.foo | not a direct argument to a likely external library call or a heuristic sink (xss) |
-| autogenerated/Xss/DomBasedXss/tst.js:316:35:316:42 | location | not a direct argument to a likely external library call or a heuristic sink (xss) |
-| autogenerated/Xss/DomBasedXss/typeahead.js:10:16:10:18 | loc | not a direct argument to a likely external library call or a heuristic sink (xss) |
-| autogenerated/Xss/DomBasedXss/typeahead.js:25:18:25:20 | val | not a direct argument to a likely external library call or a heuristic sink (xss) |

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/FilteredTruePositives.ql

@@ -20,7 +20,6 @@ import experimental.adaptivethreatmodeling.NosqlInjectionATM as NosqlInjectionAt
 import experimental.adaptivethreatmodeling.SqlInjectionATM as SqlInjectionAtm
 import experimental.adaptivethreatmodeling.TaintedPathATM as TaintedPathAtm
 import experimental.adaptivethreatmodeling.XssATM as XssAtm
-import experimental.adaptivethreatmodeling.XssThroughDomATM as XssThroughDomAtm
 
 query predicate nosqlFilteredTruePositives(DataFlow::Node endpoint, string reason) {
   endpoint instanceof NosqlInjection::Sink and
@@ -45,9 +44,3 @@ query predicate xssFilteredTruePositives(DataFlow::Node endpoint, string reason)
   reason = any(XssAtm::DomBasedXssAtmConfig cfg).getAReasonSinkExcluded(endpoint) and
   reason != "argument to modeled function"
 }
-
-query predicate xssThroughDomFilteredTruePositives(DataFlow::Node endpoint, string reason) {
-  endpoint instanceof DomBasedXss::Sink and
-  reason = any(XssThroughDomAtm::XssThroughDomAtmConfig cfg).getAReasonSinkExcluded(endpoint) and
-  reason != "argument to modeled function"
-}

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_unit_tests/ExtractEndpointDataTraining.expected

@@ -23,11 +23,6 @@ endpoints
 | index.js:15:17:15:32 | req.body.isAdmin | Xss | isExcludedFromEndToEndEvaluation | false | boolean |
 | index.js:15:17:15:32 | req.body.isAdmin | Xss | notASinkReason | LoggerMethod | string |
 | index.js:15:17:15:32 | req.body.isAdmin | Xss | sinkLabel | NotASink | string |
-| index.js:15:17:15:32 | req.body.isAdmin | XssThroughDom | hasFlowFromSource | false | boolean |
-| index.js:15:17:15:32 | req.body.isAdmin | XssThroughDom | isConstantExpression | false | boolean |
-| index.js:15:17:15:32 | req.body.isAdmin | XssThroughDom | isExcludedFromEndToEndEvaluation | false | boolean |
-| index.js:15:17:15:32 | req.body.isAdmin | XssThroughDom | notASinkReason | LoggerMethod | string |
-| index.js:15:17:15:32 | req.body.isAdmin | XssThroughDom | sinkLabel | NotASink | string |
 | index.js:20:13:20:31 | { 'isAdmin': true } | NosqlInjection | hasFlowFromSource | false | boolean |
 | index.js:20:13:20:31 | { 'isAdmin': true } | NosqlInjection | isConstantExpression | false | boolean |
 | index.js:20:13:20:31 | { 'isAdmin': true } | NosqlInjection | isExcludedFromEndToEndEvaluation | false | boolean |
@@ -60,12 +55,6 @@ endpoints
 | index.js:83:10:85:3 | {\\n    " ... ar,\\n  } | Xss | notASinkReason | ClientRequest | string |
 | index.js:83:10:85:3 | {\\n    " ... ar,\\n  } | Xss | notASinkReason | JQueryArgument | string |
 | index.js:83:10:85:3 | {\\n    " ... ar,\\n  } | Xss | sinkLabel | NotASink | string |
-| index.js:83:10:85:3 | {\\n    " ... ar,\\n  } | XssThroughDom | hasFlowFromSource | false | boolean |
-| index.js:83:10:85:3 | {\\n    " ... ar,\\n  } | XssThroughDom | isConstantExpression | false | boolean |
-| index.js:83:10:85:3 | {\\n    " ... ar,\\n  } | XssThroughDom | isExcludedFromEndToEndEvaluation | false | boolean |
-| index.js:83:10:85:3 | {\\n    " ... ar,\\n  } | XssThroughDom | notASinkReason | ClientRequest | string |
-| index.js:83:10:85:3 | {\\n    " ... ar,\\n  } | XssThroughDom | notASinkReason | JQueryArgument | string |
-| index.js:83:10:85:3 | {\\n    " ... ar,\\n  } | XssThroughDom | sinkLabel | NotASink | string |
 | index.js:84:12:84:18 | foo.bar | NosqlInjection | hasFlowFromSource | false | boolean |
 | index.js:84:12:84:18 | foo.bar | NosqlInjection | isConstantExpression | false | boolean |
 | index.js:84:12:84:18 | foo.bar | NosqlInjection | isExcludedFromEndToEndEvaluation | false | boolean |
@@ -86,11 +75,6 @@ endpoints
 | index.js:84:12:84:18 | foo.bar | Xss | isExcludedFromEndToEndEvaluation | false | boolean |
 | index.js:84:12:84:18 | foo.bar | Xss | notASinkReason | ClientRequest | string |
 | index.js:84:12:84:18 | foo.bar | Xss | sinkLabel | NotASink | string |
-| index.js:84:12:84:18 | foo.bar | XssThroughDom | hasFlowFromSource | false | boolean |
-| index.js:84:12:84:18 | foo.bar | XssThroughDom | isConstantExpression | false | boolean |
-| index.js:84:12:84:18 | foo.bar | XssThroughDom | isExcludedFromEndToEndEvaluation | false | boolean |
-| index.js:84:12:84:18 | foo.bar | XssThroughDom | notASinkReason | ClientRequest | string |
-| index.js:84:12:84:18 | foo.bar | XssThroughDom | sinkLabel | NotASink | string |
 tokenFeatures
 | index.js:9:15:9:45 | { 'isAd ... Admin } | CalleeFlexibleAccessPath | User.find |
 | index.js:9:15:9:45 | { 'isAd ... Admin } | InputAccessPathFromCallee |  |

javascript/ql/lib/semmle/javascript/ApiGraphs.qll

@@ -460,6 +460,7 @@ module API {
       this = Impl::MkClassInstance(result) or
       this = Impl::MkUse(result) or
       this = Impl::MkDef(result) or
+      this = Impl::MkAsyncFuncResult(result) or
       this = Impl::MkSyntheticCallbackArg(_, _, result)
     }
 
@@ -670,6 +671,9 @@ module API {
           cls.getAnInstanceReference() = trackDefNode(_)
         )
       } or
+      MkAsyncFuncResult(DataFlow::FunctionNode f) {
+        f = trackDefNode(_) and f.getFunction().isAsync() and hasSemantics(f)
+      } or
       MkDef(DataFlow::Node nd) { rhs(_, _, nd) } or
       MkUse(DataFlow::Node nd) { use(_, _, nd) } or
       /** A use of a TypeScript type. */
@@ -684,7 +688,8 @@ module API {
 
     class TDef = MkModuleDef or TNonModuleDef;
 
-    class TNonModuleDef = MkModuleExport or MkClassInstance or MkDef or MkSyntheticCallbackArg;
+    class TNonModuleDef =
+      MkModuleExport or MkClassInstance or MkAsyncFuncResult or MkDef or MkSyntheticCallbackArg;
 
     class TUse = MkModuleUse or MkModuleImport or MkUse or MkTypeUse;
 
@@ -735,11 +740,10 @@ module API {
             rhs = m.getAnExportedValue(prop)
           )
           or
-          exists(DataFlow::FunctionNode fn |
-            fn = pred and
-            lbl = Label::return()
-          |
-            if fn.getFunction().isAsync() then rhs = fn.getReturnNode() else rhs = fn.getAReturn()
+          exists(DataFlow::FunctionNode fn | fn = pred |
+            not fn.getFunction().isAsync() and
+            lbl = Label::return() and
+            rhs = fn.getAReturn()
           )
           or
           lbl = Label::promised() and
@@ -770,12 +774,13 @@ module API {
         )
         or
         exists(DataFlow::FunctionNode f |
-          f.getFunction().isAsync() and
-          base = MkDef(f.getReturnNode())
-        |
+          base = MkAsyncFuncResult(f) and
           lbl = Label::promised() and
           rhs = f.getAReturn()
-          or
+        )
+        or
+        exists(DataFlow::FunctionNode f |
+          base = MkAsyncFuncResult(f) and
           lbl = Label::promisedError() and
           rhs = f.getExceptionalReturn()
         )
@@ -1267,11 +1272,10 @@ module API {
       )
       or
       exists(DataFlow::Node nd, DataFlow::FunctionNode f |
-        f.getFunction().isAsync() and
         pred = MkDef(nd) and
         f = trackDefNode(nd) and
         lbl = Label::return() and
-        succ = MkDef(f.getReturnNode())
+        succ = MkAsyncFuncResult(f)
       )
       or
       exists(int bound, DataFlow::InvokeNode call |

misc/bazel/workspace.bzl

@@ -1,10 +1,10 @@
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe")
 
-_swift_prebuilt_version = "swift-5.7.1-RELEASE.44428.89"
+_swift_prebuilt_version = "swift-5.7.1-RELEASE.44356.85"
 _swift_sha_map = {
-    "Linux-X64": "1fa0b62b3a87c6528bd21b3f3fa1b32ad00e2b6ff04c20652c93c7d00c4cf517",
-    "macOS-X64": "6e1239335874cbde635ae9ca9eeb215efee4988888a75f4eda1abbcf97e4d038",
+    "Linux-X64": "c9f17e13cac89f097aaaa65b1bc32aac566c8e4f3b68fc18e970cc3ee31f4913",
+    "macOS-X64": "638ac0552e9e1ccee9e276525d1ea9454256feec20bc778efa8856fd6e506987",
 }
 
 _swift_arch_map = {
@@ -22,6 +22,10 @@ def codeql_workspace(repository_name = "codeql"):
                 _swift_prebuilt_version,
                 repo_arch,
             ),
+            patches = [
+                "@%s//swift/third_party/swift-llvm-support:patches/remove_getFallthrougDest_assert.patch" % repository_name,
+            ],
+            patch_args = ["-p1"],
             build_file = "@%s//swift/third_party/swift-llvm-support:BUILD.swift-prebuilt.bazel" % repository_name,
             sha256 = sha256,
         )

misc/scripts/models-as-data/generate_flow_model_extensions.py

@@ -173,8 +173,9 @@ Requirements: `codeql` should both appear on your path.
         else:
             negativeSummaryAddsTo = ""
         
-        return f"""# THIS FILE IS AN AUTO-GENERATED MODELS AS DATA FILE. DO NOT EDIT.
-# Definitions of models for the {self.friendlyname} framework.
+        return f""" 
+# THIS FILE IS AN AUTO-GENERATED MODELS AS DATA FILE. DO NOT EDIT.
+# Definitions of taint steps in the {self.friendlyname} framework.
 
 extensions:
 {sinkAddsTo}

python/ql/src/meta/alerts/TaintSinks.ql

@@ -1,87 +0,0 @@
-/**
- * @name Taint sinks
- * @description Sinks from TaintTracking queries.
- * @kind problem
- * @problem.severity recommendation
- * @id py/meta/alerts/taint-sinks
- * @tags meta
- * @precision very-low
- */
-
-private import python
-private import semmle.python.dataflow.new.DataFlow
-private import meta.MetaMetrics
-import semmle.python.security.dataflow.CleartextLoggingCustomizations
-import semmle.python.security.dataflow.CleartextStorageCustomizations
-import semmle.python.security.dataflow.CodeInjectionCustomizations
-import semmle.python.security.dataflow.CommandInjectionCustomizations
-import semmle.python.security.dataflow.LdapInjectionCustomizations
-import semmle.python.security.dataflow.LogInjectionCustomizations
-import semmle.python.security.dataflow.PathInjectionCustomizations
-import semmle.python.security.dataflow.PolynomialReDoSCustomizations
-import semmle.python.security.dataflow.ReflectedXSSCustomizations
-import semmle.python.security.dataflow.RegexInjectionCustomizations
-import semmle.python.security.dataflow.ServerSideRequestForgeryCustomizations
-import semmle.python.security.dataflow.SqlInjectionCustomizations
-import semmle.python.security.dataflow.StackTraceExposureCustomizations
-import semmle.python.security.dataflow.TarSlipCustomizations
-import semmle.python.security.dataflow.UnsafeDeserializationCustomizations
-import semmle.python.security.dataflow.UrlRedirectCustomizations
-import semmle.python.security.dataflow.WeakSensitiveDataHashingCustomizations
-import semmle.python.security.dataflow.XmlBombCustomizations
-import semmle.python.security.dataflow.XpathInjectionCustomizations
-import semmle.python.security.dataflow.XxeCustomizations
-
-DataFlow::Node relevantTaintSink(string kind) {
-  not result.getLocation().getFile() instanceof IgnoredFile and
-  (
-    kind = "CleartextLogging" and result instanceof CleartextLogging::Sink
-    or
-    kind = "CleartextStorage" and result instanceof CleartextStorage::Sink
-    or
-    kind = "CodeInjection" and result instanceof CodeInjection::Sink
-    or
-    kind = "CommandInjection" and result instanceof CommandInjection::Sink
-    or
-    kind = "LdapInjection (DN)" and result instanceof LdapInjection::DnSink
-    or
-    kind = "LdapInjection (Filter)" and result instanceof LdapInjection::FilterSink
-    or
-    kind = "LogInjection" and result instanceof LogInjection::Sink
-    or
-    kind = "PathInjection" and result instanceof PathInjection::Sink
-    or
-    kind = "PolynomialReDoS" and result instanceof PolynomialReDoS::Sink
-    or
-    kind = "ReflectedXss" and result instanceof ReflectedXss::Sink
-    or
-    kind = "RegexInjection" and result instanceof RegexInjection::Sink
-    or
-    kind = "ServerSideRequestForgery" and result instanceof ServerSideRequestForgery::Sink
-    or
-    kind = "SqlInjection" and result instanceof SqlInjection::Sink
-    or
-    kind = "StackTraceExposure" and result instanceof StackTraceExposure::Sink
-    or
-    kind = "TarSlip" and result instanceof TarSlip::Sink
-    or
-    kind = "UnsafeDeserialization" and result instanceof UnsafeDeserialization::Sink
-    or
-    kind = "UrlRedirect" and result instanceof UrlRedirect::Sink
-    or
-    kind = "WeakSensitiveDataHashing (NormalHashFunction)" and
-    result instanceof NormalHashFunction::Sink
-    or
-    kind = "WeakSensitiveDataHashing (ComputationallyExpensiveHashFunction)" and
-    result instanceof ComputationallyExpensiveHashFunction::Sink
-    or
-    kind = "XmlBomb" and result instanceof XmlBomb::Sink
-    or
-    kind = "XpathInjection" and result instanceof XpathInjection::Sink
-    or
-    kind = "Xxe" and result instanceof Xxe::Sink
-  )
-}
-
-from string kind
-select relevantTaintSink(kind), kind + " sink"

python/ql/src/meta/analysis-quality/CallGraph.ql

@@ -10,11 +10,7 @@
 
 import python
 import semmle.python.dataflow.new.internal.DataFlowPrivate
-import meta.MetaMetrics
 
 from DataFlowCall c, DataFlowCallableValue f
-where
-  c.getCallable() = f and
-  not c.getLocation().getFile() instanceof IgnoredFile and
-  not f.getScope().getLocation().getFile() instanceof IgnoredFile
+where c.getCallable() = f
 select c, "Call to $@", f.getScope(), f.toString()

ruby/ql/lib/change-notes/2022-11-24-json-taint-flow.md

@@ -1,4 +0,0 @@
----
- category: minorAnalysis
----
- * Taint flow is now tracked through many common JSON parsing and generation methods.

ruby/ql/lib/codeql/ruby/Frameworks.qll

@@ -24,4 +24,3 @@ private import codeql.ruby.frameworks.XmlParsing
 private import codeql.ruby.frameworks.ActionDispatch
 private import codeql.ruby.frameworks.PosixSpawn
 private import codeql.ruby.frameworks.StringFormatters
-private import codeql.ruby.frameworks.Json

ruby/ql/lib/codeql/ruby/frameworks/ActiveSupport.qll

@@ -104,17 +104,6 @@ module ActiveSupport {
 
         override predicate runsArbitraryCode() { none() }
       }
-
-      /** Flow summary for `Object#to_json`, which serializes the receiver as a JSON string. */
-      private class ToJsonSummary extends SimpleSummarizedCallable {
-        ToJsonSummary() { this = "to_json" }
-
-        override predicate propagatesFlowExt(string input, string output, boolean preservesValue) {
-          input = ["Argument[self]", "Argument[self].Element[any]"] and
-          output = "ReturnValue" and
-          preservesValue = false
-        }
-      }
     }
 
     /**
@@ -385,17 +374,4 @@ module ActiveSupport {
         ]
     }
   }
-
-  /** `ActiveSupport::JSON` */
-  module Json {
-    private class JsonSummary extends ModelInput::SummaryModelCsv {
-      override predicate row(string row) {
-        row =
-          [
-            "activesupport;;Member[ActiveSupport].Member[JSON].Method[encode,dump];Argument[0];ReturnValue;taint",
-            "activesupport;;Member[ActiveSupport].Member[JSON].Method[decode,load];Argument[0];ReturnValue;taint",
-          ]
-      }
-    }
-  }
 }

ruby/ql/lib/codeql/ruby/frameworks/Json.qll

@@ -1,22 +0,0 @@
-/** Provides modeling for the `json` gem. */
-
-private import codeql.ruby.frameworks.data.ModelsAsData
-
-/** Provides modeling for the `json` gem. */
-module Json {
-  /**
-   * Flow summaries for common `JSON` methods.
-   * Not all of these methods are strictly defined in the `json` gem.
-   * The `JSON` namespace is heavily overloaded by other JSON parsing gems such as `oj`, `json_pure`, `multi_json` etc.
-   * This summary covers common methods we've seen called on `JSON` in the wild.
-   */
-  private class JsonSummary extends ModelInput::SummaryModelCsv {
-    override predicate row(string row) {
-      row =
-        [
-          "json;;Member[JSON].Method[parse,parse!,load,restore];Argument[0];ReturnValue;taint",
-          "json;;Member[JSON].Method[generate,fast_generate,pretty_generate,dump,unparse,fast_unparse];Argument[0];ReturnValue;taint",
-        ]
-    }
-  }
-}

ruby/ql/test/TestUtilities/InlineFlowTest.qll

@@ -3,6 +3,7 @@
  *
  * Example for a test.ql:
  * ```ql
+ *  *import codeql.ruby.AST
  * import TestUtilities.InlineFlowTest
  * import PathGraph
  *

ruby/ql/test/library-tests/dataflow/local/TaintStep.expected

@@ -21,10 +21,6 @@
 | file://:0:0:0:0 | parameter position 0 of activesupport;;Member[ActionView].Member[SafeBuffer].Instance.Method[safe_concat] | file://:0:0:0:0 | [summary] to write: argument self in activesupport;;Member[ActionView].Member[SafeBuffer].Instance.Method[safe_concat] |
 | file://:0:0:0:0 | parameter position 0 of activesupport;;Member[ActionView].Member[SafeBuffer].Instance.Method[safe_concat] | file://:0:0:0:0 | [summary] to write: return (return) in activesupport;;Member[ActionView].Member[SafeBuffer].Instance.Method[safe_concat] |
 | file://:0:0:0:0 | parameter position 0 of activesupport;;Member[ActionView].Member[SafeBuffer].Method[new] | file://:0:0:0:0 | [summary] to write: return (return) in activesupport;;Member[ActionView].Member[SafeBuffer].Method[new] |
-| file://:0:0:0:0 | parameter position 0 of activesupport;;Member[ActiveSupport].Member[JSON].Method[decode,load] | file://:0:0:0:0 | [summary] to write: return (return) in activesupport;;Member[ActiveSupport].Member[JSON].Method[decode,load] |
-| file://:0:0:0:0 | parameter position 0 of activesupport;;Member[ActiveSupport].Member[JSON].Method[encode,dump] | file://:0:0:0:0 | [summary] to write: return (return) in activesupport;;Member[ActiveSupport].Member[JSON].Method[encode,dump] |
-| file://:0:0:0:0 | parameter position 0 of json;;Member[JSON].Method[generate,fast_generate,pretty_generate,dump,unparse,fast_unparse] | file://:0:0:0:0 | [summary] to write: return (return) in json;;Member[JSON].Method[generate,fast_generate,pretty_generate,dump,unparse,fast_unparse] |
-| file://:0:0:0:0 | parameter position 0 of json;;Member[JSON].Method[parse,parse!,load,restore] | file://:0:0:0:0 | [summary] to write: return (return) in json;;Member[JSON].Method[parse,parse!,load,restore] |
 | file://:0:0:0:0 | parameter position 0.. of File.join | file://:0:0:0:0 | [summary] to write: return (return) in File.join |
 | file://:0:0:0:0 | parameter self of & | file://:0:0:0:0 | [summary] read: argument self.any element in & |
 | file://:0:0:0:0 | parameter self of * | file://:0:0:0:0 | [summary] read: argument self.any element in * |

ruby/ql/test/library-tests/frameworks/active_support/ActiveSupportDataFlow.expected

@@ -189,19 +189,6 @@ edges
 | active_support.rb:303:7:303:16 | call to source :  | active_support.rb:304:19:304:19 | a :  |
 | active_support.rb:304:7:304:19 | call to json_escape :  | active_support.rb:305:8:305:8 | b |
 | active_support.rb:304:19:304:19 | a :  | active_support.rb:304:7:304:19 | call to json_escape :  |
-| active_support.rb:309:9:309:18 | call to source :  | active_support.rb:310:37:310:37 | x :  |
-| active_support.rb:310:37:310:37 | x :  | active_support.rb:310:10:310:38 | call to encode |
-| active_support.rb:314:9:314:18 | call to source :  | active_support.rb:315:37:315:37 | x :  |
-| active_support.rb:315:37:315:37 | x :  | active_support.rb:315:10:315:38 | call to decode |
-| active_support.rb:319:9:319:18 | call to source :  | active_support.rb:320:35:320:35 | x :  |
-| active_support.rb:320:35:320:35 | x :  | active_support.rb:320:10:320:36 | call to dump |
-| active_support.rb:324:9:324:18 | call to source :  | active_support.rb:325:35:325:35 | x :  |
-| active_support.rb:325:35:325:35 | x :  | active_support.rb:325:10:325:36 | call to load |
-| active_support.rb:329:9:329:18 | call to source :  | active_support.rb:330:10:330:10 | x :  |
-| active_support.rb:329:9:329:18 | call to source :  | active_support.rb:331:10:331:10 | x :  |
-| active_support.rb:330:10:330:10 | x :  | active_support.rb:332:10:332:10 | y [element 0] :  |
-| active_support.rb:331:10:331:10 | x :  | active_support.rb:331:10:331:18 | call to to_json |
-| active_support.rb:332:10:332:10 | y [element 0] :  | active_support.rb:332:10:332:18 | call to to_json |
 | hash_extensions.rb:2:14:2:24 | call to source :  | hash_extensions.rb:3:9:3:9 | h [element :a] :  |
 | hash_extensions.rb:2:14:2:24 | call to source :  | hash_extensions.rb:3:9:3:9 | h [element :a] :  |
 | hash_extensions.rb:3:9:3:9 | h [element :a] :  | hash_extensions.rb:3:9:3:24 | call to stringify_keys [element] :  |
@@ -552,24 +539,6 @@ nodes
 | active_support.rb:304:7:304:19 | call to json_escape :  | semmle.label | call to json_escape :  |
 | active_support.rb:304:19:304:19 | a :  | semmle.label | a :  |
 | active_support.rb:305:8:305:8 | b | semmle.label | b |
-| active_support.rb:309:9:309:18 | call to source :  | semmle.label | call to source :  |
-| active_support.rb:310:10:310:38 | call to encode | semmle.label | call to encode |
-| active_support.rb:310:37:310:37 | x :  | semmle.label | x :  |
-| active_support.rb:314:9:314:18 | call to source :  | semmle.label | call to source :  |
-| active_support.rb:315:10:315:38 | call to decode | semmle.label | call to decode |
-| active_support.rb:315:37:315:37 | x :  | semmle.label | x :  |
-| active_support.rb:319:9:319:18 | call to source :  | semmle.label | call to source :  |
-| active_support.rb:320:10:320:36 | call to dump | semmle.label | call to dump |
-| active_support.rb:320:35:320:35 | x :  | semmle.label | x :  |
-| active_support.rb:324:9:324:18 | call to source :  | semmle.label | call to source :  |
-| active_support.rb:325:10:325:36 | call to load | semmle.label | call to load |
-| active_support.rb:325:35:325:35 | x :  | semmle.label | x :  |
-| active_support.rb:329:9:329:18 | call to source :  | semmle.label | call to source :  |
-| active_support.rb:330:10:330:10 | x :  | semmle.label | x :  |
-| active_support.rb:331:10:331:10 | x :  | semmle.label | x :  |
-| active_support.rb:331:10:331:18 | call to to_json | semmle.label | call to to_json |
-| active_support.rb:332:10:332:10 | y [element 0] :  | semmle.label | y [element 0] :  |
-| active_support.rb:332:10:332:18 | call to to_json | semmle.label | call to to_json |
 | hash_extensions.rb:2:14:2:24 | call to source :  | semmle.label | call to source :  |
 | hash_extensions.rb:2:14:2:24 | call to source :  | semmle.label | call to source :  |
 | hash_extensions.rb:3:9:3:9 | h [element :a] :  | semmle.label | h [element :a] :  |

ruby/ql/test/library-tests/frameworks/active_support/active_support.rb

@@ -304,30 +304,3 @@ def m_json_escape
   b = json_escape a
   sink b # $hasTaintFlow=a
 end
-
-def m_json_encode
-    x = source "a"
-    sink ActiveSupport::JSON.encode(x) # $hasTaintFlow=a
-end
-
-def m_json_decode
-    x = source "a"
-    sink ActiveSupport::JSON.decode(x) # $hasTaintFlow=a
-end
-
-def m_json_dump
-    x = source "a"
-    sink ActiveSupport::JSON.dump(x) # $hasTaintFlow=a
-end
-
-def m_json_load
-    x = source "a"
-    sink ActiveSupport::JSON.load(x) # $hasTaintFlow=a
-end
-
-def m_to_json
-    x = source "a"
-    y = [x]
-    sink x.to_json # $hasTaintFlow=a
-    sink y.to_json # $hasTaintFlow=a
-end

ruby/ql/test/library-tests/frameworks/json/JsonDataFlow.expected

@@ -1,34 +0,0 @@
-failures
-edges
-| json.rb:1:17:1:26 | call to source :  | json.rb:1:6:1:27 | call to parse |
-| json.rb:2:18:2:27 | call to source :  | json.rb:2:6:2:28 | call to parse! |
-| json.rb:3:16:3:25 | call to source :  | json.rb:3:6:3:26 | call to load |
-| json.rb:4:19:4:28 | call to source :  | json.rb:4:6:4:29 | call to restore |
-| json.rb:6:20:6:29 | call to source :  | json.rb:6:6:6:30 | call to generate |
-| json.rb:7:25:7:34 | call to source :  | json.rb:7:6:7:35 | call to fast_generate |
-| json.rb:8:27:8:36 | call to source :  | json.rb:8:6:8:37 | call to pretty_generate |
-| json.rb:9:16:9:25 | call to source :  | json.rb:9:6:9:26 | call to dump |
-| json.rb:10:19:10:28 | call to source :  | json.rb:10:6:10:29 | call to unparse |
-| json.rb:11:24:11:33 | call to source :  | json.rb:11:6:11:34 | call to fast_unparse |
-nodes
-| json.rb:1:6:1:27 | call to parse | semmle.label | call to parse |
-| json.rb:1:17:1:26 | call to source :  | semmle.label | call to source :  |
-| json.rb:2:6:2:28 | call to parse! | semmle.label | call to parse! |
-| json.rb:2:18:2:27 | call to source :  | semmle.label | call to source :  |
-| json.rb:3:6:3:26 | call to load | semmle.label | call to load |
-| json.rb:3:16:3:25 | call to source :  | semmle.label | call to source :  |
-| json.rb:4:6:4:29 | call to restore | semmle.label | call to restore |
-| json.rb:4:19:4:28 | call to source :  | semmle.label | call to source :  |
-| json.rb:6:6:6:30 | call to generate | semmle.label | call to generate |
-| json.rb:6:20:6:29 | call to source :  | semmle.label | call to source :  |
-| json.rb:7:6:7:35 | call to fast_generate | semmle.label | call to fast_generate |
-| json.rb:7:25:7:34 | call to source :  | semmle.label | call to source :  |
-| json.rb:8:6:8:37 | call to pretty_generate | semmle.label | call to pretty_generate |
-| json.rb:8:27:8:36 | call to source :  | semmle.label | call to source :  |
-| json.rb:9:6:9:26 | call to dump | semmle.label | call to dump |
-| json.rb:9:16:9:25 | call to source :  | semmle.label | call to source :  |
-| json.rb:10:6:10:29 | call to unparse | semmle.label | call to unparse |
-| json.rb:10:19:10:28 | call to source :  | semmle.label | call to source :  |
-| json.rb:11:6:11:34 | call to fast_unparse | semmle.label | call to fast_unparse |
-| json.rb:11:24:11:33 | call to source :  | semmle.label | call to source :  |
-subpaths

ruby/ql/test/library-tests/frameworks/json/JsonDataFlow.ql

@@ -1,7 +0,0 @@
-/**
- * @kind path-problem
- */
-
-import TestUtilities.InlineFlowTest
-import codeql.ruby.Frameworks
-import PathGraph

ruby/ql/test/library-tests/frameworks/json/json.rb

@@ -1,11 +0,0 @@
-sink JSON.parse(source "a") # $hasTaintFlow=a
-sink JSON.parse!(source "a") # $hasTaintFlow=a
-sink JSON.load(source "a") # $hasTaintFlow=a
-sink JSON.restore(source "a") # $hasTaintFlow=a
-
-sink JSON.generate(source "a") # $hasTaintFlow=a
-sink JSON.fast_generate(source "a") # $hasTaintFlow=a
-sink JSON.pretty_generate(source "a") # $hasTaintFlow=a
-sink JSON.dump(source "a") # $hasTaintFlow=a
-sink JSON.unparse(source "a") # $hasTaintFlow=a
-sink JSON.fast_unparse(source "a") # $hasTaintFlow=a

swift/extractor/infra/CheckTypeMapping.cpp

@@ -1,42 +0,0 @@
-// this is a sanity check that the hierarchy in swift is mapped by the mapping in
-// SwiftTypesToTagsMap.def to the hierarchy in schema.py
-// We rely on that so that the LabelStore will preserve correct typing.
-// For a class Derived: Base we could store the label for a Derived* pointer, and then fetch the
-// label for the same pointer as Base*. If the mapping did not preserve inheritance, we would end up
-// using a trap key of the DB type associated with Derived in a position expecting the incompatible
-// DB type associated with Base.
-
-#include "swift/extractor/infra/SwiftTagTraits.h"
-
-using namespace codeql;
-
-#define CHECK(KIND, TYPE, PARENT)                                                              \
-  static_assert(std::is_same_v<TrapTagOf<swift::TYPE##KIND>, void> ||                          \
-                    std::is_base_of_v<TrapTagOf<swift::PARENT>, TrapTagOf<swift::TYPE##KIND>>, \
-                "Tag of " #PARENT " must be a base of the tag of " #TYPE #KIND);
-#define CHECK_CONCRETE(KIND, TYPE, PARENT)                                                       \
-  CHECK(KIND, TYPE, PARENT)                                                                      \
-  static_assert(                                                                                 \
-      std::is_same_v<TrapTagOf<swift::TYPE##KIND>, void> ||                                      \
-          std::is_base_of_v<TrapTagOf<swift::TYPE##KIND>, ConcreteTrapTagOf<swift::TYPE##KIND>>, \
-      "Tag of " #TYPE #KIND " must be a base of its concrete tag");
-
-#define STMT(CLASS, PARENT) CHECK_CONCRETE(Stmt, CLASS, PARENT)
-#define ABSTRACT_STMT(CLASS, PARENT) CHECK(Stmt, CLASS, PARENT)
-#include <swift/AST/StmtNodes.def>
-
-#define EXPR(CLASS, PARENT) CHECK_CONCRETE(Expr, CLASS, PARENT)
-#define ABSTRACT_EXPR(CLASS, PARENT) CHECK(Expr, CLASS, PARENT)
-#include <swift/AST/ExprNodes.def>
-
-#define DECL(CLASS, PARENT) CHECK_CONCRETE(Decl, CLASS, PARENT)
-#define ABSTRACT_DECL(CLASS, PARENT) CHECK(Decl, CLASS, PARENT)
-#include <swift/AST/DeclNodes.def>
-
-#define PATTERN(CLASS, PARENT) CHECK_CONCRETE(Pattern, CLASS, PARENT)
-#define ABSTRACT_PATTERN(CLASS, PARENT) CHECK(Pattern, CLASS, PARENT)
-#include <swift/AST/PatternNodes.def>
-
-#define TYPE(CLASS, PARENT) CHECK_CONCRETE(Type, CLASS, PARENT)
-#define ABSTRACT_TYPE(CLASS, PARENT) CHECK(Type, CLASS, PARENT)
-#include <swift/AST/TypeNodes.def>

swift/extractor/infra/SwiftDispatcher.h

@@ -179,16 +179,16 @@ class SwiftDispatcher {
   // it actually gets emitted to handle recursive cases such as recursive calls, or recursive type
   // declarations
   template <typename E, typename... Args, std::enable_if_t<IsStorable<E>>* = nullptr>
-  TrapLabel<ConcreteTrapTagOf<E>> assignNewLabel(const E& e, Args&&... args) {
+  TrapLabelOf<E> assignNewLabel(const E& e, Args&&... args) {
     assert(waitingForNewLabel == Store::Handle{e} && "assignNewLabel called on wrong entity");
-    auto label = trap.createLabel<ConcreteTrapTagOf<E>>(std::forward<Args>(args)...);
+    auto label = trap.createLabel<TrapTagOf<E>>(std::forward<Args>(args)...);
     store.insert(e, label);
     waitingForNewLabel = std::monostate{};
     return label;
   }
 
   template <typename E, typename... Args, std::enable_if_t<IsStorable<E*>>* = nullptr>
-  TrapLabel<ConcreteTrapTagOf<E>> assignNewLabel(const E& e, Args&&... args) {
+  TrapLabelOf<E> assignNewLabel(const E& e, Args&&... args) {
     return assignNewLabel(&e, std::forward<Args>(args)...);
   }
 
@@ -215,6 +215,17 @@ class SwiftDispatcher {
   // Emits a Location TRAP entry and attaches it to a `Locatable` trap label
   template <typename Locatable>
   void attachLocation(Locatable* locatable, TrapLabel<LocatableTag> locatableLabel) {
+    // Swift 5.7.1 causing a crash when the Swift compiler uses .back() of and empty
+    // getPatternList() when getting the source location.
+    // So far this only observed with the entities coming from precompiled modules, so they
+    // should not have locations anyway.
+    if constexpr (std::is_base_of<swift::Decl, Locatable>::value) {
+      if (auto* decl = llvm::dyn_cast<swift::PatternBindingDecl>(locatable)) {
+        if (decl->getPatternList().empty()) {
+          return;
+        }
+      }
+    }
     attachLocation(locatable->getStartLoc(), locatable->getEndLoc(), locatableLabel);
   }
 

swift/extractor/infra/SwiftTagTraits.h

@@ -1,320 +1,102 @@
 #pragma once
 
 // This file implements the mapping needed by the API defined in the TrapTagTraits.h, so that
-// TrapTagOf/TrapLabelOf/TrapClassOf provide the tags/labels for specific swift entity types.
+// TrapTagOf/TrapLabelOf provide the tags/labels for specific swift entity types.
 #include <filesystem>
+#include <swift/AST/ASTVisitor.h>
 #include "swift/extractor/trap/TrapTagTraits.h"
 #include "swift/extractor/trap/generated/TrapTags.h"
-#include <swift/AST/ASTNode.h>
-#include <swift/AST/Decl.h>
-#include <swift/AST/Expr.h>
-#include <swift/AST/Stmt.h>
-#include <swift/AST/Pattern.h>
-#include <swift/AST/TypeRepr.h>
-#include <swift/AST/Type.h>
 
 namespace codeql {
 
-#define MAP(TYPE, TAG)                \
+// OverloadSetRefExpr is collapsed with its only derived class OverloadedDeclRefExpr
+using OverloadSetRefExprTag = OverloadedDeclRefExprTag;
+
+// We don't really expect to see the following in extraction. Mapping these tags to void effectively
+// ignores all elements of that class (with a message).
+
+// only generated for code editing
+using CodeCompletionExprTag = void;
+using EditorPlaceholderExprTag = void;
+// not present after the Sema phase
+using ArrowExprTag = void;
+// experimental variadic generics, implemented only in the frontend for now, thus not compilable
+using PackExprTag = void;
+using PackTypeTag = void;
+using ReifyPackExprTag = void;
+using PackExpansionTypeTag = void;
+using SequenceArchetypeTypeTag = void;
+// Placeholder types appear in ambiguous types but are anyway transformed to UnresolvedType
+using PlaceholderTypeTag = void;
+// SIL types that cannot really appear in the frontend run
+using SILBlockStorageTypeTag = void;
+using SILBoxTypeTag = void;
+using SILFunctionTypeTag = void;
+using SILTokenTypeTag = void;
+// This is created during type checking and is only used for constraint checking
+using TypeVariableTypeTag = void;
+
+using ABISafeConversionExprTag = AbiSafeConversionExprTag;
+
+#define MAP_TYPE_TO_TAG(TYPE, TAG)    \
   template <>                         \
   struct detail::ToTagFunctor<TYPE> { \
     using type = TAG;                 \
-  };
-#define MAP_CONCRETE(TYPE, TAG)                \
-  template <>                                  \
-  struct detail::ToTagConcreteOverride<TYPE> { \
-    using type = TAG;                          \
-  };
+  }
+#define MAP_TAG(TYPE) MAP_TYPE_TO_TAG(swift::TYPE, TYPE##Tag)
+#define MAP_SUBTAG(TYPE, PARENT)                                                              \
+  MAP_TAG(TYPE);                                                                              \
+  static_assert(std::is_same_v<TYPE##Tag, void> || std::is_base_of_v<PARENT##Tag, TYPE##Tag>, \
+                #PARENT "Tag must be a base of " #TYPE "Tag");
 
-// clang-format off
-// use indentation to recreate all involved type hierarchies
-MAP(std::filesystem::path, DbFileTag)
-MAP(swift::StmtCondition, StmtConditionTag)
-MAP(swift::StmtConditionElement, ConditionElementTag)
-MAP(swift::CaseLabelItem, CaseLabelItemTag)
+#define OVERRIDE_TAG(TYPE, TAG)               \
+  template <>                                 \
+  struct detail::ToTagOverride<swift::TYPE> { \
+    using type = TAG;                         \
+  };                                          \
+  static_assert(std::is_base_of_v<TYPE##Tag, TAG>, "override is not a subtag");
 
-MAP(swift::Stmt, StmtTag)
-  MAP(swift::BraceStmt, BraceStmtTag)
-  MAP(swift::ReturnStmt, ReturnStmtTag)
-  MAP(swift::YieldStmt, YieldStmtTag)
-  MAP(swift::DeferStmt, DeferStmtTag)
-  MAP(swift::LabeledStmt, LabeledStmtTag)
-    MAP(swift::LabeledConditionalStmt, LabeledConditionalStmtTag)
-      MAP(swift::IfStmt, IfStmtTag)
-      MAP(swift::GuardStmt, GuardStmtTag)
-      MAP(swift::WhileStmt, WhileStmtTag)
-    MAP(swift::DoStmt, DoStmtTag)
-    MAP(swift::DoCatchStmt, DoCatchStmtTag)
-    MAP(swift::RepeatWhileStmt, RepeatWhileStmtTag)
-    MAP(swift::ForEachStmt, ForEachStmtTag)
-    MAP(swift::SwitchStmt, SwitchStmtTag)
-  MAP(swift::CaseStmt, CaseStmtTag)
-  MAP(swift::BreakStmt, BreakStmtTag)
-  MAP(swift::ContinueStmt, ContinueStmtTag)
-  MAP(swift::FallthroughStmt, FallthroughStmtTag)
-  MAP(swift::FailStmt, FailStmtTag)
-  MAP(swift::ThrowStmt, ThrowStmtTag)
-  MAP(swift::PoundAssertStmt, PoundAssertStmtTag)
+MAP_TAG(Stmt);
+MAP_TAG(StmtCondition);
+MAP_TYPE_TO_TAG(swift::StmtConditionElement, ConditionElementTag);
+MAP_TAG(CaseLabelItem);
+#define ABSTRACT_STMT(CLASS, PARENT) MAP_SUBTAG(CLASS##Stmt, PARENT)
+#define STMT(CLASS, PARENT) ABSTRACT_STMT(CLASS, PARENT)
+#include <swift/AST/StmtNodes.def>
 
-MAP(swift::Argument, ArgumentTag)
-MAP(swift::Expr, ExprTag)
-  MAP(swift::ErrorExpr, ErrorExprTag)
-  MAP(swift::LiteralExpr, LiteralExprTag)
-    MAP(swift::NilLiteralExpr, NilLiteralExprTag)
-    MAP(swift::BuiltinLiteralExpr, BuiltinLiteralExprTag)
-      MAP(swift::BooleanLiteralExpr, BooleanLiteralExprTag)
-      MAP(swift::NumberLiteralExpr, NumberLiteralExprTag)
-        MAP(swift::IntegerLiteralExpr, IntegerLiteralExprTag)
-        MAP(swift::FloatLiteralExpr, FloatLiteralExprTag)
-      MAP(swift::StringLiteralExpr, StringLiteralExprTag)
-      MAP(swift::MagicIdentifierLiteralExpr, MagicIdentifierLiteralExprTag)
-    MAP(swift::InterpolatedStringLiteralExpr, InterpolatedStringLiteralExprTag)
-    MAP(swift::RegexLiteralExpr, RegexLiteralExprTag)
-    MAP(swift::ObjectLiteralExpr, ObjectLiteralExprTag)
-  MAP(swift::DiscardAssignmentExpr, DiscardAssignmentExprTag)
-  MAP(swift::DeclRefExpr, DeclRefExprTag)
-  MAP(swift::SuperRefExpr, SuperRefExprTag)
-  MAP(swift::TypeExpr, TypeExprTag)
-  MAP(swift::OtherConstructorDeclRefExpr, OtherConstructorDeclRefExprTag)
-  MAP(swift::DotSyntaxBaseIgnoredExpr, DotSyntaxBaseIgnoredExprTag)
-  MAP(swift::OverloadSetRefExpr, OverloadedDeclRefExprTag)  // collapsed with its only derived class OverloadedDeclRefExpr
-    MAP(swift::OverloadedDeclRefExpr, OverloadedDeclRefExprTag)
-  MAP(swift::UnresolvedDeclRefExpr, UnresolvedDeclRefExprTag)
-  MAP(swift::LookupExpr, LookupExprTag)
-    MAP(swift::MemberRefExpr, MemberRefExprTag)
-    MAP(swift::SubscriptExpr, SubscriptExprTag)
-    MAP(swift::DynamicLookupExpr, DynamicLookupExprTag)
-      MAP(swift::DynamicMemberRefExpr, DynamicMemberRefExprTag)
-      MAP(swift::DynamicSubscriptExpr, DynamicSubscriptExprTag)
-  MAP(swift::UnresolvedSpecializeExpr, UnresolvedSpecializeExprTag)
-  MAP(swift::UnresolvedMemberExpr, UnresolvedMemberExprTag)
-  MAP(swift::UnresolvedDotExpr, UnresolvedDotExprTag)
-  MAP(swift::SequenceExpr, SequenceExprTag)
-  MAP(swift::IdentityExpr, IdentityExprTag)
-    MAP(swift::ParenExpr, ParenExprTag)
-    MAP(swift::DotSelfExpr, DotSelfExprTag)
-    MAP(swift::AwaitExpr, AwaitExprTag)
-    MAP(swift::UnresolvedMemberChainResultExpr, UnresolvedMemberChainResultExprTag)
-  MAP(swift::AnyTryExpr, AnyTryExprTag)
-    MAP(swift::TryExpr, TryExprTag)
-    MAP(swift::ForceTryExpr, ForceTryExprTag)
-    MAP(swift::OptionalTryExpr, OptionalTryExprTag)
-  MAP(swift::TupleExpr, TupleExprTag)
-  MAP(swift::CollectionExpr, CollectionExprTag)
-    MAP(swift::ArrayExpr, ArrayExprTag)
-    MAP(swift::DictionaryExpr, DictionaryExprTag)
-  MAP(swift::KeyPathApplicationExpr, KeyPathApplicationExprTag)
-  MAP(swift::TupleElementExpr, TupleElementExprTag)
-  MAP(swift::CaptureListExpr, CaptureListExprTag)
-  MAP(swift::AbstractClosureExpr, AbstractClosureExprTag)
-    MAP(swift::ClosureExpr, ClosureExprTag)
-    MAP(swift::AutoClosureExpr, AutoClosureExprTag)
-  MAP(swift::InOutExpr, InOutExprTag)
-  MAP(swift::VarargExpansionExpr, VarargExpansionExprTag)
-  MAP(swift::DynamicTypeExpr, DynamicTypeExprTag)
-  MAP(swift::RebindSelfInConstructorExpr, RebindSelfInConstructorExprTag)
-  MAP(swift::OpaqueValueExpr, OpaqueValueExprTag)
-  MAP(swift::PropertyWrapperValuePlaceholderExpr, PropertyWrapperValuePlaceholderExprTag)
-  MAP(swift::AppliedPropertyWrapperExpr, AppliedPropertyWrapperExprTag)
-  MAP(swift::DefaultArgumentExpr, DefaultArgumentExprTag)
-  MAP(swift::BindOptionalExpr, BindOptionalExprTag)
-  MAP(swift::OptionalEvaluationExpr, OptionalEvaluationExprTag)
-  MAP(swift::ForceValueExpr, ForceValueExprTag)
-  MAP(swift::OpenExistentialExpr, OpenExistentialExprTag)
-  MAP(swift::MakeTemporarilyEscapableExpr, MakeTemporarilyEscapableExprTag)
-  MAP(swift::ApplyExpr, ApplyExprTag)
-    MAP(swift::CallExpr, CallExprTag)
-    MAP(swift::PrefixUnaryExpr, PrefixUnaryExprTag)
-    MAP(swift::PostfixUnaryExpr, PostfixUnaryExprTag)
-    MAP(swift::BinaryExpr, BinaryExprTag)
-    MAP(swift::SelfApplyExpr, SelfApplyExprTag)
-      MAP(swift::DotSyntaxCallExpr, DotSyntaxCallExprTag)
-      MAP(swift::ConstructorRefCallExpr, ConstructorRefCallExprTag)
-  MAP(swift::ImplicitConversionExpr, ImplicitConversionExprTag)
-    MAP(swift::LoadExpr, LoadExprTag)
-    MAP(swift::DestructureTupleExpr, DestructureTupleExprTag)
-    MAP(swift::UnresolvedTypeConversionExpr, UnresolvedTypeConversionExprTag)
-    MAP(swift::FunctionConversionExpr, FunctionConversionExprTag)
-    MAP(swift::CovariantFunctionConversionExpr, CovariantFunctionConversionExprTag)
-    MAP(swift::CovariantReturnConversionExpr, CovariantReturnConversionExprTag)
-    MAP(swift::MetatypeConversionExpr, MetatypeConversionExprTag)
-    MAP(swift::CollectionUpcastConversionExpr, CollectionUpcastConversionExprTag)
-    MAP(swift::ErasureExpr, ErasureExprTag)
-    MAP(swift::AnyHashableErasureExpr, AnyHashableErasureExprTag)
-    MAP(swift::BridgeToObjCExpr, BridgeToObjCExprTag)
-    MAP(swift::BridgeFromObjCExpr, BridgeFromObjCExprTag)
-    MAP(swift::ConditionalBridgeFromObjCExpr, ConditionalBridgeFromObjCExprTag)
-    MAP(swift::DerivedToBaseExpr, DerivedToBaseExprTag)
-    MAP(swift::ArchetypeToSuperExpr, ArchetypeToSuperExprTag)
-    MAP(swift::InjectIntoOptionalExpr, InjectIntoOptionalExprTag)
-    MAP(swift::ClassMetatypeToObjectExpr, ClassMetatypeToObjectExprTag)
-    MAP(swift::ExistentialMetatypeToObjectExpr, ExistentialMetatypeToObjectExprTag)
-    MAP(swift::ProtocolMetatypeToObjectExpr, ProtocolMetatypeToObjectExprTag)
-    MAP(swift::InOutToPointerExpr, InOutToPointerExprTag)
-    MAP(swift::ArrayToPointerExpr, ArrayToPointerExprTag)
-    MAP(swift::StringToPointerExpr, StringToPointerExprTag)
-    MAP(swift::PointerToPointerExpr, PointerToPointerExprTag)
-    MAP(swift::ForeignObjectConversionExpr, ForeignObjectConversionExprTag)
-    MAP(swift::UnevaluatedInstanceExpr, UnevaluatedInstanceExprTag)
-    MAP(swift::UnderlyingToOpaqueExpr, UnderlyingToOpaqueExprTag)
-    MAP(swift::DifferentiableFunctionExpr, DifferentiableFunctionExprTag)
-    MAP(swift::LinearFunctionExpr, LinearFunctionExprTag)
-    MAP(swift::DifferentiableFunctionExtractOriginalExpr, DifferentiableFunctionExtractOriginalExprTag)
-    MAP(swift::LinearFunctionExtractOriginalExpr, LinearFunctionExtractOriginalExprTag)
-    MAP(swift::LinearToDifferentiableFunctionExpr, LinearToDifferentiableFunctionExprTag)
-    MAP(swift::ReifyPackExpr, void)  // experimental variadic generics
-    MAP(swift::ABISafeConversionExpr, AbiSafeConversionExprTag)  // different acronym convention
-  MAP(swift::ExplicitCastExpr, ExplicitCastExprTag)
-    MAP(swift::CheckedCastExpr, CheckedCastExprTag)
-      MAP(swift::ForcedCheckedCastExpr, ForcedCheckedCastExprTag)
-      MAP(swift::ConditionalCheckedCastExpr, ConditionalCheckedCastExprTag)
-      MAP(swift::IsExpr, IsExprTag)
-    MAP(swift::CoerceExpr, CoerceExprTag)
-  MAP(swift::ArrowExpr, void)  // not present after the Sema phase
-  MAP(swift::IfExpr, IfExprTag)
-  MAP(swift::EnumIsCaseExpr, EnumIsCaseExprTag)
-  MAP(swift::AssignExpr, AssignExprTag)
-  MAP(swift::CodeCompletionExpr, void) // only generated for code editing
-  MAP(swift::UnresolvedPatternExpr, UnresolvedPatternExprTag)
-  MAP(swift::LazyInitializerExpr, LazyInitializerExprTag)
-  MAP(swift::EditorPlaceholderExpr, void)  // only generated for code editing
-  MAP(swift::ObjCSelectorExpr, ObjCSelectorExprTag)
-  MAP(swift::KeyPathExpr, KeyPathExprTag)
-  MAP(swift::KeyPathDotExpr, KeyPathDotExprTag)
-  MAP(swift::OneWayExpr, OneWayExprTag)
-  MAP(swift::TapExpr, TapExprTag)
-  MAP(swift::PackExpr, void)  // experimental variadic generics
+MAP_TAG(Expr);
+MAP_TAG(Argument);
+#define ABSTRACT_EXPR(CLASS, PARENT) MAP_SUBTAG(CLASS##Expr, PARENT)
+#define EXPR(CLASS, PARENT) ABSTRACT_EXPR(CLASS, PARENT)
+#include <swift/AST/ExprNodes.def>
 
-MAP(swift::Decl, DeclTag)
-  MAP(swift::ValueDecl, ValueDeclTag)
-    MAP(swift::TypeDecl, TypeDeclTag)
-      MAP(swift::GenericTypeDecl, GenericTypeDeclTag)
-        MAP(swift::NominalTypeDecl, NominalTypeDeclTag)
-          MAP(swift::EnumDecl, EnumDeclTag)
-          MAP(swift::StructDecl, StructDeclTag)
-          MAP(swift::ClassDecl, ClassDeclTag)
-          MAP(swift::ProtocolDecl, ProtocolDeclTag)
-        MAP(swift::OpaqueTypeDecl, OpaqueTypeDeclTag)
-        MAP(swift::TypeAliasDecl, TypeAliasDeclTag)
-      MAP(swift::AbstractTypeParamDecl, AbstractTypeParamDeclTag)
-        MAP(swift::GenericTypeParamDecl, GenericTypeParamDeclTag)
-        MAP(swift::AssociatedTypeDecl, AssociatedTypeDeclTag)
-      MAP(swift::ModuleDecl, ModuleDeclTag)
-    MAP(swift::AbstractStorageDecl, AbstractStorageDeclTag)
-      MAP(swift::VarDecl, VarDeclTag)
-        MAP_CONCRETE(swift::VarDecl, ConcreteVarDeclTag)
-        MAP(swift::ParamDecl, ParamDeclTag)
-      MAP(swift::SubscriptDecl, SubscriptDeclTag)
-    MAP(swift::AbstractFunctionDecl, AbstractFunctionDeclTag)
-      MAP(swift::ConstructorDecl, ConstructorDeclTag)
-      MAP(swift::DestructorDecl, DestructorDeclTag)
-      MAP(swift::FuncDecl, FuncDeclTag)
-        MAP_CONCRETE(swift::FuncDecl, ConcreteFuncDeclTag)
-        MAP(swift::AccessorDecl, AccessorDeclTag)
-    MAP(swift::EnumElementDecl, EnumElementDeclTag)
-  MAP(swift::ExtensionDecl, ExtensionDeclTag)
-  MAP(swift::TopLevelCodeDecl, TopLevelCodeDeclTag)
-  MAP(swift::ImportDecl, ImportDeclTag)
-  MAP(swift::IfConfigDecl, IfConfigDeclTag)
-  MAP(swift::PoundDiagnosticDecl, PoundDiagnosticDeclTag)
-  MAP(swift::PrecedenceGroupDecl, PrecedenceGroupDeclTag)
-  MAP(swift::MissingMemberDecl, MissingMemberDeclTag)
-  MAP(swift::PatternBindingDecl, PatternBindingDeclTag)
-  MAP(swift::EnumCaseDecl, EnumCaseDeclTag)
-  MAP(swift::OperatorDecl, OperatorDeclTag)
-    MAP(swift::InfixOperatorDecl, InfixOperatorDeclTag)
-    MAP(swift::PrefixOperatorDecl, PrefixOperatorDeclTag)
-    MAP(swift::PostfixOperatorDecl, PostfixOperatorDeclTag)
+MAP_TAG(Decl);
+#define ABSTRACT_DECL(CLASS, PARENT) MAP_SUBTAG(CLASS##Decl, PARENT)
+#define DECL(CLASS, PARENT) ABSTRACT_DECL(CLASS, PARENT)
+#include <swift/AST/DeclNodes.def>
 
-MAP(swift::Pattern, PatternTag)
-  MAP(swift::ParenPattern, ParenPatternTag)
-  MAP(swift::TuplePattern, TuplePatternTag)
-  MAP(swift::NamedPattern, NamedPatternTag)
-  MAP(swift::AnyPattern, AnyPatternTag)
-  MAP(swift::TypedPattern, TypedPatternTag)
-  MAP(swift::BindingPattern, BindingPatternTag)
-  MAP(swift::IsPattern, IsPatternTag)
-  MAP(swift::EnumElementPattern, EnumElementPatternTag)
-  MAP(swift::OptionalSomePattern, OptionalSomePatternTag)
-  MAP(swift::BoolPattern, BoolPatternTag)
-  MAP(swift::ExprPattern, ExprPatternTag)
+MAP_TAG(Pattern);
+#define ABSTRACT_PATTERN(CLASS, PARENT) MAP_SUBTAG(CLASS##Pattern, PARENT)
+#define PATTERN(CLASS, PARENT) ABSTRACT_PATTERN(CLASS, PARENT)
+#include <swift/AST/PatternNodes.def>
 
-MAP(swift::TypeRepr, TypeReprTag)
+MAP_TAG(TypeRepr);
+
+MAP_TYPE_TO_TAG(swift::TypeBase, TypeTag);
+#define ABSTRACT_TYPE(CLASS, PARENT) MAP_SUBTAG(CLASS##Type, PARENT)
+#define TYPE(CLASS, PARENT) ABSTRACT_TYPE(CLASS, PARENT)
+#include <swift/AST/TypeNodes.def>
+
+OVERRIDE_TAG(FuncDecl, ConcreteFuncDeclTag);
+OVERRIDE_TAG(VarDecl, ConcreteVarDeclTag);
+
+MAP_TYPE_TO_TAG(std::filesystem::path, DbFileTag);
+
+#undef MAP_TAG
+#undef MAP_SUBTAG
+#undef MAP_TYPE_TO_TAG
+#undef OVERRIDE_TAG
+
+// All the other macros defined here are undefined by the .def files
 
-MAP(swift::Type, TypeTag)
-MAP(swift::TypeBase, TypeTag)
-  MAP(swift::ErrorType, ErrorTypeTag)
-  MAP(swift::UnresolvedType, UnresolvedTypeTag)
-  MAP(swift::PlaceholderType, void)  // appears in ambiguous types but are then transformed to UnresolvedType
-  MAP(swift::BuiltinType, BuiltinTypeTag)
-    MAP(swift::AnyBuiltinIntegerType, AnyBuiltinIntegerTypeTag)
-      MAP(swift::BuiltinIntegerType, BuiltinIntegerTypeTag)
-      MAP(swift::BuiltinIntegerLiteralType, BuiltinIntegerLiteralTypeTag)
-    MAP(swift::BuiltinExecutorType, BuiltinExecutorTypeTag)
-    MAP(swift::BuiltinFloatType, BuiltinFloatTypeTag)
-    MAP(swift::BuiltinJobType, BuiltinJobTypeTag)
-    MAP(swift::BuiltinRawPointerType, BuiltinRawPointerTypeTag)
-    MAP(swift::BuiltinRawUnsafeContinuationType, BuiltinRawUnsafeContinuationTypeTag)
-    MAP(swift::BuiltinNativeObjectType, BuiltinNativeObjectTypeTag)
-    MAP(swift::BuiltinBridgeObjectType, BuiltinBridgeObjectTypeTag)
-    MAP(swift::BuiltinUnsafeValueBufferType, BuiltinUnsafeValueBufferTypeTag)
-    MAP(swift::BuiltinDefaultActorStorageType, BuiltinDefaultActorStorageTypeTag)
-    MAP(swift::BuiltinVectorType, BuiltinVectorTypeTag)
-  MAP(swift::TupleType, TupleTypeTag)
-  MAP(swift::ReferenceStorageType, ReferenceStorageTypeTag)
-  MAP(swift::WeakStorageType, WeakStorageTypeTag)
-  MAP(swift::UnownedStorageType, UnownedStorageTypeTag)
-  MAP(swift::UnmanagedStorageType, UnmanagedStorageTypeTag)
-  MAP(swift::AnyGenericType, AnyGenericTypeTag)
-    MAP(swift::NominalOrBoundGenericNominalType, NominalOrBoundGenericNominalTypeTag)
-      MAP(swift::NominalType, NominalTypeTag)
-        MAP(swift::EnumType, EnumTypeTag)
-        MAP(swift::StructType, StructTypeTag)
-        MAP(swift::ClassType, ClassTypeTag)
-        MAP(swift::ProtocolType, ProtocolTypeTag)
-      MAP(swift::BoundGenericType, BoundGenericTypeTag)
-        MAP(swift::BoundGenericClassType, BoundGenericClassTypeTag)
-        MAP(swift::BoundGenericEnumType, BoundGenericEnumTypeTag)
-        MAP(swift::BoundGenericStructType, BoundGenericStructTypeTag)
-    MAP(swift::UnboundGenericType, UnboundGenericTypeTag)
-  MAP(swift::AnyMetatypeType, AnyMetatypeTypeTag)
-    MAP(swift::MetatypeType, MetatypeTypeTag)
-    MAP(swift::ExistentialMetatypeType, ExistentialMetatypeTypeTag)
-  MAP(swift::ModuleType, ModuleTypeTag)
-  MAP(swift::DynamicSelfType, DynamicSelfTypeTag)
-  MAP(swift::SubstitutableType, SubstitutableTypeTag)
-    MAP(swift::ArchetypeType, ArchetypeTypeTag)
-      MAP(swift::PrimaryArchetypeType, PrimaryArchetypeTypeTag)
-      MAP(swift::OpaqueTypeArchetypeType, OpaqueTypeArchetypeTypeTag)
-      MAP(swift::OpenedArchetypeType, OpenedArchetypeTypeTag)
-      MAP(swift::SequenceArchetypeType, void)  // experimental variadic generics
-    MAP(swift::GenericTypeParamType, GenericTypeParamTypeTag)
-  MAP(swift::DependentMemberType, DependentMemberTypeTag)
-  MAP(swift::AnyFunctionType, AnyFunctionTypeTag)
-    MAP(swift::FunctionType, FunctionTypeTag)
-    MAP(swift::GenericFunctionType, GenericFunctionTypeTag)
-  MAP(swift::SILFunctionType, void)  // SIL types cannot really appear in the frontend run)
-  MAP(swift::SILBlockStorageType, void)  // SIL types cannot really appear in the frontend run)
-  MAP(swift::SILBoxType, void)  // SIL types cannot really appear in the frontend run)
-  MAP(swift::SILTokenType, void)  // SIL types cannot really appear in the frontend run)
-  MAP(swift::ProtocolCompositionType, ProtocolCompositionTypeTag)
-  MAP(swift::ParameterizedProtocolType, ParameterizedProtocolTypeTag)
-  MAP(swift::ExistentialType, ExistentialTypeTag)
-  MAP(swift::LValueType, LValueTypeTag)
-  MAP(swift::InOutType, InOutTypeTag)
-  MAP(swift::PackType, void)  // experimental variadic generics
-  MAP(swift::PackExpansionType, void)  // experimental variadic generics
-  MAP(swift::TypeVariableType, void)  // created during type checking and only used for constraint checking
-  MAP(swift::SugarType, SugarTypeTag)
-    MAP(swift::ParenType, ParenTypeTag)
-    MAP(swift::TypeAliasType, TypeAliasTypeTag)
-    MAP(swift::SyntaxSugarType, SyntaxSugarTypeTag)
-      MAP(swift::UnarySyntaxSugarType, UnarySyntaxSugarTypeTag)
-        MAP(swift::ArraySliceType, ArraySliceTypeTag)
-        MAP(swift::OptionalType, OptionalTypeTag)
-        MAP(swift::VariadicSequenceType, VariadicSequenceTypeTag)
-      MAP(swift::DictionaryType, DictionaryTypeTag)
-// clang-format on
-#undef MAP
-#undef MAP_CONCRETE
 }  // namespace codeql

swift/extractor/translators/DeclTranslator.h

@@ -65,9 +65,9 @@ class DeclTranslator : public AstTranslatorBase<DeclTranslator> {
                                codeql::AbstractStorageDecl& entry);
 
   template <typename D>
-  auto createNamedEntry(const D& decl) {
-    std::optional<TrapClassOf<D>> entry;
+  std::optional<TrapClassOf<D>> createNamedEntry(const D& decl) {
     auto id = dispatcher.assignNewLabel(decl, mangledName(decl));
+    std::optional<TrapClassOf<D>> entry;
     if (dispatcher.shouldEmitDeclBody(decl)) {
       entry.emplace(id);
       fillDecl(decl, *entry);
@@ -76,7 +76,7 @@ class DeclTranslator : public AstTranslatorBase<DeclTranslator> {
   }
 
   template <typename D>
-  auto createEntry(const D& decl) {
+  TrapClassOf<D> createEntry(const D& decl) {
     TrapClassOf<D> entry{dispatcher.template assignNewLabel(decl)};
     fillDecl(decl, entry);
     return entry;

swift/extractor/translators/TranslatorBase.h

@@ -75,7 +75,7 @@ enum class TranslatorPolicy {
 #define DEFINE_VISIT(KIND, CLASS, PARENT)                                            \
  public:                                                                             \
   static constexpr TranslatorPolicy getPolicyFor##CLASS##KIND() {                    \
-    if constexpr (std::is_same_v<TrapTagOf<swift::CLASS##KIND>, void>) {             \
+    if constexpr (std::is_same_v<CLASS##KIND##Tag, void>) {                          \
       return TranslatorPolicy::ignore;                                               \
     } else if constexpr (detail::HasTranslate##CLASS##KIND<CrtpSubclass>::value) {   \
       return TranslatorPolicy::translate;                                            \

swift/extractor/trap/TrapTagTraits.h

@@ -15,7 +15,7 @@ struct ToTagFunctor;
 
 // can be instantiated to override the default mapping for special cases
 template <typename T>
-struct ToTagConcreteOverride : ToTagFunctor<T> {};
+struct ToTagOverride : ToTagFunctor<T> {};
 
 // must be instantiated to map trap labels to the corresponding generated binding trap entry
 template <typename Label>
@@ -28,19 +28,15 @@ struct ToTrapClassFunctor;
 
 template <typename T>
 using TrapTagOf =
-    typename detail::ToTagFunctor<std::remove_const_t<std::remove_pointer_t<T>>>::type;
-
-template <typename T>
-using ConcreteTrapTagOf =
-    typename detail::ToTagConcreteOverride<std::remove_const_t<std::remove_pointer_t<T>>>::type;
+    typename detail::ToTagOverride<std::remove_const_t<std::remove_pointer_t<T>>>::type;
 
 template <typename T>
 using TrapLabelOf = TrapLabel<TrapTagOf<T>>;
 
 template <typename T>
-using BindingTrapOf = typename detail::ToBindingTrapFunctor<TrapLabel<ConcreteTrapTagOf<T>>>::type;
+using BindingTrapOf = typename detail::ToBindingTrapFunctor<TrapLabelOf<T>>::type;
 
 template <typename T>
-using TrapClassOf = typename detail::ToTrapClassFunctor<ConcreteTrapTagOf<T>>::type;
+using TrapClassOf = typename detail::ToTrapClassFunctor<TrapTagOf<T>>::type;
 
 }  // namespace codeql

swift/integration-tests/.gitignore

@@ -1,9 +1,9 @@
 .DS_Store
 .build
 Packages
+*.xcodeproj
 xcuserdata/
 DerivedData/
 .swiftpm/xcode/package.xcworkspace/contents.xcworkspacedata
 *.actual
 db
-*.swiftmodule

swift/integration-tests/osx-only/hello-xcode/Files.expected

@@ -1,2 +0,0 @@
-| codeql-swift-autobuild-test/AppDelegate.swift:0:0:0:0 | codeql-swift-autobuild-test/AppDelegate.swift |
-| file://:0:0:0:0 |  |

swift/integration-tests/osx-only/hello-xcode/Files.ql

@@ -1,4 +0,0 @@
-import swift
-
-from File f
-select f

swift/integration-tests/osx-only/hello-xcode/codeql-swift-autobuild-test.xcodeproj/project.pbxproj

@@ -1,310 +0,0 @@
-// !$*UTF8*$!
-{
-	archiveVersion = 1;
-	classes = {
-	};
-	objectVersion = 56;
-	objects = {
-
-/* Begin PBXBuildFile section */
-		46D4896F291B98000029E1E2 /* AppDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = 46D4896E291B98000029E1E2 /* AppDelegate.swift */; };
-/* End PBXBuildFile section */
-
-/* Begin PBXFileReference section */
-		46D4896B291B98000029E1E2 /* codeql-swift-autobuild-test.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = "codeql-swift-autobuild-test.app"; sourceTree = BUILT_PRODUCTS_DIR; };
-		46D4896E291B98000029E1E2 /* AppDelegate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppDelegate.swift; sourceTree = "<group>"; };
-/* End PBXFileReference section */
-
-/* Begin PBXFrameworksBuildPhase section */
-		46D48968291B98000029E1E2 /* Frameworks */ = {
-			isa = PBXFrameworksBuildPhase;
-			buildActionMask = 2147483647;
-			files = (
-			);
-			runOnlyForDeploymentPostprocessing = 0;
-		};
-/* End PBXFrameworksBuildPhase section */
-
-/* Begin PBXGroup section */
-		46D48962291B98000029E1E2 = {
-			isa = PBXGroup;
-			children = (
-				46D4896D291B98000029E1E2 /* codeql-swift-autobuild-test */,
-				46D4896C291B98000029E1E2 /* Products */,
-			);
-			sourceTree = "<group>";
-		};
-		46D4896C291B98000029E1E2 /* Products */ = {
-			isa = PBXGroup;
-			children = (
-				46D4896B291B98000029E1E2 /* codeql-swift-autobuild-test.app */,
-			);
-			name = Products;
-			sourceTree = "<group>";
-		};
-		46D4896D291B98000029E1E2 /* codeql-swift-autobuild-test */ = {
-			isa = PBXGroup;
-			children = (
-				46D4896E291B98000029E1E2 /* AppDelegate.swift */,
-			);
-			path = "codeql-swift-autobuild-test";
-			sourceTree = "<group>";
-		};
-/* End PBXGroup section */
-
-/* Begin PBXNativeTarget section */
-		46D4896A291B98000029E1E2 /* codeql-swift-autobuild-test */ = {
-			isa = PBXNativeTarget;
-			buildConfigurationList = 46D4897A291B98020029E1E2 /* Build configuration list for PBXNativeTarget "codeql-swift-autobuild-test" */;
-			buildPhases = (
-				46D48967291B98000029E1E2 /* Sources */,
-				46D48968291B98000029E1E2 /* Frameworks */,
-				46D48969291B98000029E1E2 /* Resources */,
-			);
-			buildRules = (
-			);
-			dependencies = (
-			);
-			name = "codeql-swift-autobuild-test";
-			productName = "codeql-swift-autobuild-test";
-			productReference = 46D4896B291B98000029E1E2 /* codeql-swift-autobuild-test.app */;
-			productType = "com.apple.product-type.application";
-		};
-/* End PBXNativeTarget section */
-
-/* Begin PBXProject section */
-		46D48963291B98000029E1E2 /* Project object */ = {
-			isa = PBXProject;
-			attributes = {
-				BuildIndependentTargetsInParallel = 1;
-				LastSwiftUpdateCheck = 1400;
-				LastUpgradeCheck = 1400;
-				TargetAttributes = {
-					46D4896A291B98000029E1E2 = {
-						CreatedOnToolsVersion = 14.0;
-					};
-				};
-			};
-			buildConfigurationList = 46D48966291B98000029E1E2 /* Build configuration list for PBXProject "codeql-swift-autobuild-test" */;
-			compatibilityVersion = "Xcode 14.0";
-			developmentRegion = en;
-			hasScannedForEncodings = 0;
-			knownRegions = (
-				en,
-			);
-			mainGroup = 46D48962291B98000029E1E2;
-			productRefGroup = 46D4896C291B98000029E1E2 /* Products */;
-			projectDirPath = "";
-			projectRoot = "";
-			targets = (
-				46D4896A291B98000029E1E2 /* codeql-swift-autobuild-test */,
-			);
-		};
-/* End PBXProject section */
-
-/* Begin PBXResourcesBuildPhase section */
-		46D48969291B98000029E1E2 /* Resources */ = {
-			isa = PBXResourcesBuildPhase;
-			buildActionMask = 2147483647;
-			files = (
-			);
-			runOnlyForDeploymentPostprocessing = 0;
-		};
-/* End PBXResourcesBuildPhase section */
-
-/* Begin PBXSourcesBuildPhase section */
-		46D48967291B98000029E1E2 /* Sources */ = {
-			isa = PBXSourcesBuildPhase;
-			buildActionMask = 2147483647;
-			files = (
-				46D4896F291B98000029E1E2 /* AppDelegate.swift in Sources */,
-			);
-			runOnlyForDeploymentPostprocessing = 0;
-		};
-/* End PBXSourcesBuildPhase section */
-
-/* Begin XCBuildConfiguration section */
-		46D48978291B98020029E1E2 /* Debug */ = {
-			isa = XCBuildConfiguration;
-			buildSettings = {
-				ALWAYS_SEARCH_USER_PATHS = NO;
-				CLANG_ANALYZER_NONNULL = YES;
-				CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
-				CLANG_CXX_LANGUAGE_STANDARD = "gnu++20";
-				CLANG_ENABLE_MODULES = YES;
-				CLANG_ENABLE_OBJC_ARC = YES;
-				CLANG_ENABLE_OBJC_WEAK = YES;
-				CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;
-				CLANG_WARN_BOOL_CONVERSION = YES;
-				CLANG_WARN_COMMA = YES;
-				CLANG_WARN_CONSTANT_CONVERSION = YES;
-				CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;
-				CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;
-				CLANG_WARN_DOCUMENTATION_COMMENTS = YES;
-				CLANG_WARN_EMPTY_BODY = YES;
-				CLANG_WARN_ENUM_CONVERSION = YES;
-				CLANG_WARN_INFINITE_RECURSION = YES;
-				CLANG_WARN_INT_CONVERSION = YES;
-				CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;
-				CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;
-				CLANG_WARN_OBJC_LITERAL_CONVERSION = YES;
-				CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
-				CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES;
-				CLANG_WARN_RANGE_LOOP_ANALYSIS = YES;
-				CLANG_WARN_STRICT_PROTOTYPES = YES;
-				CLANG_WARN_SUSPICIOUS_MOVE = YES;
-				CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;
-				CLANG_WARN_UNREACHABLE_CODE = YES;
-				CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
-				COPY_PHASE_STRIP = NO;
-				DEBUG_INFORMATION_FORMAT = dwarf;
-				ENABLE_STRICT_OBJC_MSGSEND = YES;
-				ENABLE_TESTABILITY = YES;
-				GCC_C_LANGUAGE_STANDARD = gnu11;
-				GCC_DYNAMIC_NO_PIC = NO;
-				GCC_NO_COMMON_BLOCKS = YES;
-				GCC_OPTIMIZATION_LEVEL = 0;
-				GCC_PREPROCESSOR_DEFINITIONS = (
-					"DEBUG=1",
-					"$(inherited)",
-				);
-				GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
-				GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
-				GCC_WARN_UNDECLARED_SELECTOR = YES;
-				GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
-				GCC_WARN_UNUSED_FUNCTION = YES;
-				GCC_WARN_UNUSED_VARIABLE = YES;
-				MACOSX_DEPLOYMENT_TARGET = 11.0;
-				MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE;
-				MTL_FAST_MATH = YES;
-				ONLY_ACTIVE_ARCH = YES;
-				SDKROOT = macosx;
-				SWIFT_ACTIVE_COMPILATION_CONDITIONS = DEBUG;
-				SWIFT_OPTIMIZATION_LEVEL = "-Onone";
-			};
-			name = Debug;
-		};
-		46D48979291B98020029E1E2 /* Release */ = {
-			isa = XCBuildConfiguration;
-			buildSettings = {
-				ALWAYS_SEARCH_USER_PATHS = NO;
-				CLANG_ANALYZER_NONNULL = YES;
-				CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
-				CLANG_CXX_LANGUAGE_STANDARD = "gnu++20";
-				CLANG_ENABLE_MODULES = YES;
-				CLANG_ENABLE_OBJC_ARC = YES;
-				CLANG_ENABLE_OBJC_WEAK = YES;
-				CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;
-				CLANG_WARN_BOOL_CONVERSION = YES;
-				CLANG_WARN_COMMA = YES;
-				CLANG_WARN_CONSTANT_CONVERSION = YES;
-				CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;
-				CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;
-				CLANG_WARN_DOCUMENTATION_COMMENTS = YES;
-				CLANG_WARN_EMPTY_BODY = YES;
-				CLANG_WARN_ENUM_CONVERSION = YES;
-				CLANG_WARN_INFINITE_RECURSION = YES;
-				CLANG_WARN_INT_CONVERSION = YES;
-				CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;
-				CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;
-				CLANG_WARN_OBJC_LITERAL_CONVERSION = YES;
-				CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
-				CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES;
-				CLANG_WARN_RANGE_LOOP_ANALYSIS = YES;
-				CLANG_WARN_STRICT_PROTOTYPES = YES;
-				CLANG_WARN_SUSPICIOUS_MOVE = YES;
-				CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;
-				CLANG_WARN_UNREACHABLE_CODE = YES;
-				CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
-				COPY_PHASE_STRIP = NO;
-				DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
-				ENABLE_NS_ASSERTIONS = NO;
-				ENABLE_STRICT_OBJC_MSGSEND = YES;
-				GCC_C_LANGUAGE_STANDARD = gnu11;
-				GCC_NO_COMMON_BLOCKS = YES;
-				GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
-				GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
-				GCC_WARN_UNDECLARED_SELECTOR = YES;
-				GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
-				GCC_WARN_UNUSED_FUNCTION = YES;
-				GCC_WARN_UNUSED_VARIABLE = YES;
-				MACOSX_DEPLOYMENT_TARGET = 11.0;
-				MTL_ENABLE_DEBUG_INFO = NO;
-				MTL_FAST_MATH = YES;
-				SDKROOT = macosx;
-				SWIFT_COMPILATION_MODE = wholemodule;
-				SWIFT_OPTIMIZATION_LEVEL = "-O";
-			};
-			name = Release;
-		};
-		46D4897B291B98020029E1E2 /* Debug */ = {
-			isa = XCBuildConfiguration;
-			buildSettings = {
-				ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor;
-				CODE_SIGN_STYLE = Automatic;
-				COMBINE_HIDPI_IMAGES = YES;
-				CURRENT_PROJECT_VERSION = 1;
-				GENERATE_INFOPLIST_FILE = YES;
-				INFOPLIST_KEY_NSHumanReadableCopyright = "";
-				INFOPLIST_KEY_NSMainStoryboardFile = Main;
-				INFOPLIST_KEY_NSPrincipalClass = NSApplication;
-				LD_RUNPATH_SEARCH_PATHS = (
-					"$(inherited)",
-					"@executable_path/../Frameworks",
-				);
-				MARKETING_VERSION = 1.0;
-				PRODUCT_BUNDLE_IDENTIFIER = "com.github.codeql-swift-autobuild-test";
-				PRODUCT_NAME = "$(TARGET_NAME)";
-				SWIFT_EMIT_LOC_STRINGS = YES;
-				SWIFT_VERSION = 5.0;
-			};
-			name = Debug;
-		};
-		46D4897C291B98020029E1E2 /* Release */ = {
-			isa = XCBuildConfiguration;
-			buildSettings = {
-				ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor;
-				CODE_SIGN_STYLE = Automatic;
-				COMBINE_HIDPI_IMAGES = YES;
-				CURRENT_PROJECT_VERSION = 1;
-				GENERATE_INFOPLIST_FILE = YES;
-				INFOPLIST_KEY_NSHumanReadableCopyright = "";
-				INFOPLIST_KEY_NSMainStoryboardFile = Main;
-				INFOPLIST_KEY_NSPrincipalClass = NSApplication;
-				LD_RUNPATH_SEARCH_PATHS = (
-					"$(inherited)",
-					"@executable_path/../Frameworks",
-				);
-				MARKETING_VERSION = 1.0;
-				PRODUCT_BUNDLE_IDENTIFIER = "com.github.codeql-swift-autobuild-test";
-				PRODUCT_NAME = "$(TARGET_NAME)";
-				SWIFT_EMIT_LOC_STRINGS = YES;
-				SWIFT_VERSION = 5.0;
-			};
-			name = Release;
-		};
-/* End XCBuildConfiguration section */
-
-/* Begin XCConfigurationList section */
-		46D48966291B98000029E1E2 /* Build configuration list for PBXProject "codeql-swift-autobuild-test" */ = {
-			isa = XCConfigurationList;
-			buildConfigurations = (
-				46D48978291B98020029E1E2 /* Debug */,
-				46D48979291B98020029E1E2 /* Release */,
-			);
-			defaultConfigurationIsVisible = 0;
-			defaultConfigurationName = Release;
-		};
-		46D4897A291B98020029E1E2 /* Build configuration list for PBXNativeTarget "codeql-swift-autobuild-test" */ = {
-			isa = XCConfigurationList;
-			buildConfigurations = (
-				46D4897B291B98020029E1E2 /* Debug */,
-				46D4897C291B98020029E1E2 /* Release */,
-			);
-			defaultConfigurationIsVisible = 0;
-			defaultConfigurationName = Release;
-		};
-/* End XCConfigurationList section */
-	};
-	rootObject = 46D48963291B98000029E1E2 /* Project object */;
-}

swift/integration-tests/osx-only/hello-xcode/codeql-swift-autobuild-test/AppDelegate.swift

@@ -1,4 +0,0 @@
-import Cocoa
-
-@main
-class AppDelegate: NSObject, NSApplicationDelegate {}

swift/integration-tests/osx-only/hello-xcode/test.py

@@ -1,9 +0,0 @@
-from create_database_utils import *
-
-run_codeql_database_create([
-    'xcodebuild clean',
-    'xcodebuild build '
-    '-project codeql-swift-autobuild-test.xcodeproj '
-    '-target codeql-swift-autobuild-test '
-    'CODE_SIGNING_REQUIRED=NO CODE_SIGNING_ALLOWED=NO',
-], lang='swift', keep_trap=True)

swift/integration-tests/posix-only/frontend-invocations/Files.expected

@@ -3,6 +3,4 @@
 | C.swift:0:0:0:0 | C.swift |
 | D.swift:0:0:0:0 | D.swift |
 | E.swift:0:0:0:0 | E.swift |
-| F1.swift:0:0:0:0 | F1.swift |
-| F2.swift:0:0:0:0 | F2.swift |
 | file://:0:0:0:0 |  |

swift/integration-tests/posix-only/frontend-invocations/Makefile

@@ -0,0 +1,15 @@
+OS=$(shell uname)
+ifeq ($(OS),Darwin)
+SDK=-sdk $(shell xcrun -show-sdk-path)
+FRONTEND=$(shell xcrun -find swift-frontend)
+else
+SDK=""
+FRONTEND=swiftc
+endif
+
+all:
+	$(FRONTEND) -frontend -c A.swift $(SDK)
+	$(FRONTEND) -frontend -c B.swift -o B.o $(SDK)
+	$(FRONTEND) -frontend -c -primary-file C.swift $(SDK)
+	$(FRONTEND) -frontend -c -primary-file D.swift -o D.o $(SDK)
+	$(FRONTEND) -frontend -c -primary-file E.swift Esup.swift -o E.o $(SDK)

swift/integration-tests/posix-only/frontend-invocations/build.sh

@@ -1,18 +0,0 @@
-#!/bin/bash
-
-if [[ "$(uname)" == Darwin ]]; then
-  SDK="-sdk $(xcrun -show-sdk-path)"
-  FRONTEND="$(xcrun -find swift-frontend)"
-else
-  SDK=""
-  FRONTEND="swift-frontend"
-fi
-
-$FRONTEND -frontend -c A.swift $SDK
-$FRONTEND -frontend -c B.swift -o B.o $SDK
-$FRONTEND -frontend -c -primary-file C.swift $SDK
-$FRONTEND -frontend -c -primary-file D.swift -o D.o $SDK
-$FRONTEND -frontend -c -primary-file E.swift Esup.swift -o E.o $SDK
-$FRONTEND -frontend -emit-module -primary-file F1.swift F2.swift -module-name F -o F1.swiftmodule $SDK
-$FRONTEND -frontend -emit-module F1.swift -primary-file F2.swift -module-name F -o F2.swiftmodule $SDK
-$FRONTEND -merge-modules F1.swiftmodule F2.swiftmodule -o F.swiftmodule $SDK

swift/integration-tests/posix-only/frontend-invocations/test.py

@@ -1,5 +1,5 @@
 from create_database_utils import *
 
 run_codeql_database_create([
-    './build.sh',
+    'make',
 ], lang='swift')

swift/ql/lib/codeql/swift/elements/expr/ArrowExpr.qll

@@ -0,0 +1,5 @@
+private import codeql.swift.generated.expr.ArrowExpr
+
+class ArrowExpr extends Generated::ArrowExpr {
+  override string toString() { result = "... -> ..." }
+}

swift/third_party/swift-llvm-support/patches/remove_getFallthrougDest_assert.patch

@@ -0,0 +1,11 @@
+diff -ru a/include/swift/AST/Stmt.h b/include/swift/AST/Stmt.h
+--- a/include/swift/AST/Stmt.h	2022-09-21 12:56:54.000000000 +0200
++++ b/include/swift/AST/Stmt.h	2022-11-04 14:39:18.407971007 +0100
+@@ -920,7 +920,6 @@
+   /// Get the CaseStmt block to which the fallthrough transfers control.
+   /// Set during Sema.
+   CaseStmt *getFallthroughDest() const {
+-    assert(FallthroughDest && "fallthrough dest is not set until Sema");
+     return FallthroughDest;
+   }
+   void setFallthroughDest(CaseStmt *C) {