Merge pull request #11738 from michaelnebel/go/maddataextensions

Go: Models as Data using extensions.

go/ql/lib/ext/archive_tar.model.yml

@@ -0,0 +1,6 @@
+extensions:
+  - addsTo:
+      pack: codeql/go-all
+      extensible: summaryModel
+    data:
+      - ["archive/tar", "", True, "FileInfoHeader", "", "", "Argument[0]", "ReturnValue[0]", "taint", "manual"]

go/ql/lib/ext/builtin.model.yml

@@ -0,0 +1,7 @@
+extensions:
+  - addsTo:
+      pack: codeql/go-all
+      extensible: summaryModel
+    data:
+      - ["", "", False, "append", "", "", "Argument[0].ArrayElement", "ReturnValue.ArrayElement", "value", "manual"]
+      - ["", "", False, "append", "", "", "Argument[1]", "ReturnValue.ArrayElement", "value", "manual"]

go/ql/lib/ext/dummy.model.yml

@@ -0,0 +1,14 @@
+extensions:
+  # Make sure that the extensible model predicates are at least defined as empty.
+  - addsTo:
+      pack: codeql/go-all
+      extensible: sourceModel
+    data: []
+  - addsTo:
+      pack: codeql/go-all
+      extensible: sinkModel
+    data: []
+  - addsTo:
+      pack: codeql/go-all
+      extensible: summaryModel
+    data: []

go/ql/lib/ext/net_http.model.yml

@@ -0,0 +1,13 @@
+extensions:
+  - addsTo:
+      pack: codeql/go-all
+      extensible: sourceModel
+    data:
+      - ["net/http", "Request", True, "Cookie", "", "", "ReturnValue[0]", "remote", "manual"]
+      - ["net/http", "Request", True, "Cookies", "", "", "ReturnValue.ArrayElement", "remote", "manual"]
+      - ["net/http", "Request", True, "FormFile", "", "", "ReturnValue[0..1]", "remote", "manual"]
+      - ["net/http", "Request", True, "FormValue", "", "", "ReturnValue", "remote", "manual"]
+      - ["net/http", "Request", True, "MultipartReader", "", "", "ReturnValue[0]", "remote", "manual"]
+      - ["net/http", "Request", True, "PostFormValue", "", "", "ReturnValue", "remote", "manual"]
+      - ["net/http", "Request", True, "Referer", "", "", "ReturnValue", "remote", "manual"]
+      - ["net/http", "Request", True, "UserAgent", "", "", "ReturnValue", "remote", "manual"]

go/ql/lib/qlpack.yml

@@ -7,4 +7,5 @@ library: true
 upgrades: upgrades
 dependencies:
   codeql/tutorial: ${workspace}
-
+dataExtensions:
+  - ext/*.model.yml

go/ql/lib/semmle/go/dataflow/ExternalFlow.qll

@@ -1,19 +1,20 @@
 /**
  * INTERNAL use only. This is an experimental API subject to change without notice.
  *
- * Provides classes and predicates for dealing with flow models specified in CSV format.
+ * Provides classes and predicates for dealing with MaD flow models specified
+ * in data extensions and CSV format.
  *
  * The CSV specification has the following columns:
  * - Sources:
- *   `namespace; type; subtypes; name; signature; ext; output; kind`
+ *   `package; type; subtypes; name; signature; ext; output; kind; provenance`
  * - Sinks:
- *   `namespace; type; subtypes; name; signature; ext; input; kind`
+ *   `package; type; subtypes; name; signature; ext; input; kind; provenance`
  * - Summaries:
- *   `namespace; type; subtypes; name; signature; ext; input; output; kind`
+ *   `package; type; subtypes; name; signature; ext; input; output; kind; provenance`
  *
  * The interpretation of a row is similar to API-graphs with a left-to-right
  * reading.
- * 1. The `namespace` column selects a package.
+ * 1. The `package` column selects a package.
  * 2. The `type` column selects a type within that package.
  * 3. The `subtypes` is a boolean that indicates whether to jump to an
  *    arbitrary subtype of that type.
@@ -61,6 +62,7 @@
  */
 
 private import go
+private import ExternalFlowExtensions as Extensions
 private import internal.DataFlowPrivate
 private import internal.FlowSummaryImpl::Private::External
 private import internal.FlowSummaryImplSpecific
@@ -75,145 +77,44 @@ private module Frameworks {
   private import semmle.go.frameworks.Stdlib
 }
 
-private class BuiltinModel extends SummaryModelCsv {
-  override predicate row(string row) {
-    row =
-      [
-        ";;false;append;;;Argument[0].ArrayElement;ReturnValue.ArrayElement;value",
-        ";;false;append;;;Argument[1];ReturnValue.ArrayElement;value"
-      ]
-  }
-}
-
-/**
- * A unit class for adding additional source model rows.
- *
- * Extend this class to add additional source definitions.
- */
-class SourceModelCsv extends Unit {
-  /** Holds if `row` specifies a source definition. */
-  abstract predicate row(string row);
-}
-
-/**
- * A unit class for adding additional sink model rows.
- *
- * Extend this class to add additional sink definitions.
- */
-class SinkModelCsv extends Unit {
-  /** Holds if `row` specifies a sink definition. */
-  abstract predicate row(string row);
-}
-
-/**
- * A unit class for adding additional summary model rows.
- *
- * Extend this class to add additional flow summary definitions.
- */
-class SummaryModelCsv extends Unit {
-  /** Holds if `row` specifies a summary definition. */
-  abstract predicate row(string row);
-}
-
-/** Holds if `row` is a source model. */
-predicate sourceModel(string row) { any(SourceModelCsv s).row(row) }
-
-/** Holds if `row` is a sink model. */
-predicate sinkModel(string row) { any(SinkModelCsv s).row(row) }
-
-/** Holds if `row` is a summary model. */
-predicate summaryModel(string row) { any(SummaryModelCsv s).row(row) }
-
 /** Holds if a source model exists for the given parameters. */
-predicate sourceModel(
-  string namespace, string type, boolean subtypes, string name, string signature, string ext,
-  string output, string kind, string provenance
-) {
-  exists(string row |
-    sourceModel(row) and
-    row.splitAt(";", 0) = namespace and
-    row.splitAt(";", 1) = type and
-    row.splitAt(";", 2) = subtypes.toString() and
-    subtypes = [true, false] and
-    row.splitAt(";", 3) = name and
-    row.splitAt(";", 4) = signature and
-    row.splitAt(";", 5) = ext and
-    row.splitAt(";", 6) = output and
-    row.splitAt(";", 7) = kind and
-    provenance = "manual"
-  )
-}
+predicate sourceModel = Extensions::sourceModel/9;
 
 /** Holds if a sink model exists for the given parameters. */
-predicate sinkModel(
-  string namespace, string type, boolean subtypes, string name, string signature, string ext,
-  string input, string kind, string provenance
-) {
-  exists(string row |
-    sinkModel(row) and
-    row.splitAt(";", 0) = namespace and
-    row.splitAt(";", 1) = type and
-    row.splitAt(";", 2) = subtypes.toString() and
-    subtypes = [true, false] and
-    row.splitAt(";", 3) = name and
-    row.splitAt(";", 4) = signature and
-    row.splitAt(";", 5) = ext and
-    row.splitAt(";", 6) = input and
-    row.splitAt(";", 7) = kind and
-    provenance = "manual"
-  )
-}
+predicate sinkModel = Extensions::sinkModel/9;
 
 /** Holds if a summary model exists for the given parameters. */
-predicate summaryModel(
-  string namespace, string type, boolean subtypes, string name, string signature, string ext,
-  string input, string output, string kind, string provenance
-) {
-  exists(string row |
-    summaryModel(row) and
-    row.splitAt(";", 0) = namespace and
-    row.splitAt(";", 1) = type and
-    row.splitAt(";", 2) = subtypes.toString() and
-    subtypes = [true, false] and
-    row.splitAt(";", 3) = name and
-    row.splitAt(";", 4) = signature and
-    row.splitAt(";", 5) = ext and
-    row.splitAt(";", 6) = input and
-    row.splitAt(";", 7) = output and
-    row.splitAt(";", 8) = kind
-  ) and
-  provenance = "manual"
-}
+predicate summaryModel = Extensions::summaryModel/10;
 
-/** Holds if `package` have CSV framework coverage. */
-private predicate packageHasCsvCoverage(string package) {
+/** Holds if `package` have MaD framework coverage. */
+private predicate packageHasMaDCoverage(string package) {
   sourceModel(package, _, _, _, _, _, _, _, _) or
   sinkModel(package, _, _, _, _, _, _, _, _) or
   summaryModel(package, _, _, _, _, _, _, _, _, _)
 }
 
 /**
- * Holds if `package` and `subpkg` have CSV framework coverage and `subpkg`
+ * Holds if `package` and `subpkg` have MaD framework coverage and `subpkg`
  * is a subpackage of `package`.
  */
 private predicate packageHasASubpackage(string package, string subpkg) {
-  packageHasCsvCoverage(package) and
-  packageHasCsvCoverage(subpkg) and
+  packageHasMaDCoverage(package) and
+  packageHasMaDCoverage(subpkg) and
   subpkg.prefix(subpkg.indexOf(".")) = package
 }
 
 /**
- * Holds if `package` has CSV framework coverage and it is not a subpackage of
- * any other package with CSV framework coverage.
+ * Holds if `package` has MaD framework coverage and it is not a subpackage of
+ * any other package with MaD framework coverage.
  */
 private predicate canonicalPackage(string package) {
-  packageHasCsvCoverage(package) and not packageHasASubpackage(_, package)
+  packageHasMaDCoverage(package) and not packageHasASubpackage(_, package)
 }
 
 /**
- * Holds if `package` and `subpkg` have CSV framework coverage, `subpkg` is a
+ * Holds if `package` and `subpkg` have MaD framework coverage, `subpkg` is a
  * subpackage of `package` (or they are the same), and `package` is not a
- * subpackage of any other package with CSV framework coverage.
+ * subpackage of any other package with MaD framework coverage.
  */
 private predicate canonicalPackageHasASubpackage(string package, string subpkg) {
   canonicalPackage(package) and
@@ -221,9 +122,9 @@ private predicate canonicalPackageHasASubpackage(string package, string subpkg)
 }
 
 /**
- * Holds if CSV framework coverage of `package` is `n` api endpoints of the
+ * Holds if MaD framework coverage of `package` is `n` api endpoints of the
  * kind `(kind, part)`, and `pkgs` is the number of subpackages of `package`
- * which have CSV framework coverage (including `package` itself).
+ * which have MaD framework coverage (including `package` itself).
  */
 predicate modelCoverage(string package, int pkgs, string kind, string part, int n) {
   pkgs = strictcount(string subpkg | canonicalPackageHasASubpackage(package, subpkg)) and
@@ -254,8 +155,8 @@ predicate modelCoverage(string package, int pkgs, string kind, string part, int
   )
 }
 
-/** Provides a query predicate to check the CSV data for validation errors. */
-module CsvValidation {
+/** Provides a query predicate to check the MaD models for validation errors. */
+module ModelValidation {
   private string getInvalidModelInput() {
     exists(string pred, AccessPath input, string part |
       sinkModel(_, _, _, _, _, _, input, _, _) and pred = "sink"
@@ -288,57 +189,25 @@ module CsvValidation {
   }
 
   private string getInvalidModelKind() {
-    exists(string row, string kind | summaryModel(row) |
-      kind = row.splitAt(";", 8) and
+    exists(string kind | summaryModel(_, _, _, _, _, _, _, _, kind, _) |
       not kind = ["taint", "value"] and
       result = "Invalid kind \"" + kind + "\" in summary model."
     )
   }
 
-  private string getInvalidModelSubtype() {
-    exists(string pred, string row |
-      sourceModel(row) and pred = "source"
-      or
-      sinkModel(row) and pred = "sink"
-      or
-      summaryModel(row) and pred = "summary"
-    |
-      exists(string b |
-        b = row.splitAt(";", 2) and
-        not b = ["true", "false"] and
-        result = "Invalid boolean \"" + b + "\" in " + pred + " model."
-      )
-    )
-  }
-
-  private string getInvalidModelColumnCount() {
-    exists(string pred, string row, int expect |
-      sourceModel(row) and expect = 8 and pred = "source"
-      or
-      sinkModel(row) and expect = 8 and pred = "sink"
-      or
-      summaryModel(row) and expect = 9 and pred = "summary"
-    |
-      exists(int cols |
-        cols = 1 + max(int n | exists(row.splitAt(";", n))) and
-        cols != expect and
-        result =
-          "Wrong number of columns in " + pred + " model row, expected " + expect + ", got " + cols +
-            "."
-      )
-    )
-  }
-
   private string getInvalidModelSignature() {
-    exists(string pred, string namespace, string type, string name, string signature, string ext |
-      sourceModel(namespace, type, _, name, signature, ext, _, _, _) and pred = "source"
-      or
-      sinkModel(namespace, type, _, name, signature, ext, _, _, _) and pred = "sink"
-      or
-      summaryModel(namespace, type, _, name, signature, ext, _, _, _, _) and pred = "summary"
+    exists(
+      string pred, string package, string type, string name, string signature, string ext,
+      string provenance
     |
-      not namespace.regexpMatch("[a-zA-Z0-9_\\./]*") and
-      result = "Dubious namespace \"" + namespace + "\" in " + pred + " model."
+      sourceModel(package, type, _, name, signature, ext, _, _, provenance) and pred = "source"
+      or
+      sinkModel(package, type, _, name, signature, ext, _, _, provenance) and pred = "sink"
+      or
+      summaryModel(package, type, _, name, signature, ext, _, _, _, provenance) and pred = "summary"
+    |
+      not package.regexpMatch("[a-zA-Z0-9_\\./]*") and
+      result = "Dubious package \"" + package + "\" in " + pred + " model."
       or
       not type.regexpMatch("[a-zA-Z0-9_\\$<>]*") and
       result = "Dubious type \"" + type + "\" in " + pred + " model."
@@ -351,26 +220,29 @@ module CsvValidation {
       or
       not ext.regexpMatch("|Annotated") and
       result = "Unrecognized extra API graph element \"" + ext + "\" in " + pred + " model."
+      or
+      not provenance = ["manual", "generated"] and
+      result = "Unrecognized provenance description \"" + provenance + "\" in " + pred + " model."
     )
   }
 
-  /** Holds if some row in a CSV-based flow model appears to contain typos. */
+  /** Holds if some row in a MaD flow model appears to contain typos. */
   query predicate invalidModelRow(string msg) {
     msg =
       [
         getInvalidModelSignature(), getInvalidModelInput(), getInvalidModelOutput(),
-        getInvalidModelSubtype(), getInvalidModelColumnCount(), getInvalidModelKind()
+        getInvalidModelKind()
       ]
   }
 }
 
 pragma[nomagic]
 private predicate elementSpec(
-  string namespace, string type, boolean subtypes, string name, string signature, string ext
+  string package, string type, boolean subtypes, string name, string signature, string ext
 ) {
-  sourceModel(namespace, type, subtypes, name, signature, ext, _, _, _) or
-  sinkModel(namespace, type, subtypes, name, signature, ext, _, _, _) or
-  summaryModel(namespace, type, subtypes, name, signature, ext, _, _, _, _)
+  sourceModel(package, type, subtypes, name, signature, ext, _, _, _) or
+  sinkModel(package, type, subtypes, name, signature, ext, _, _, _) or
+  summaryModel(package, type, subtypes, name, signature, ext, _, _, _, _)
 }
 
 private string paramsStringPart(Function f, int i) {
@@ -466,7 +338,7 @@ predicate parseContent(string component, DataFlow::Content content) {
 cached
 private module Cached {
   /**
-   * Holds if `node` is specified as a source with the given kind in a CSV flow
+   * Holds if `node` is specified as a source with the given kind in a MaD flow
    * model.
    */
   cached
@@ -475,7 +347,7 @@ private module Cached {
   }
 
   /**
-   * Holds if `node` is specified as a sink with the given kind in a CSV flow
+   * Holds if `node` is specified as a sink with the given kind in a MaD flow
    * model.
    */
   cached

go/ql/lib/semmle/go/dataflow/ExternalFlowExtensions.qll

@@ -0,0 +1,27 @@
+/**
+ * This module provides extensible predicates for defining MaD models.
+ */
+
+/**
+ * Holds if a source model exists for the given parameters.
+ */
+extensible predicate sourceModel(
+  string package, string type, boolean subtypes, string name, string signature, string ext,
+  string output, string kind, string provenance
+);
+
+/**
+ * Holds if a sink model exists for the given parameters.
+ */
+extensible predicate sinkModel(
+  string package, string type, boolean subtypes, string name, string signature, string ext,
+  string input, string kind, string provenance
+);
+
+/**
+ * Holds if a summary model exists for the given parameters.
+ */
+extensible predicate summaryModel(
+  string package, string type, boolean subtypes, string name, string signature, string ext,
+  string input, string output, string kind, string provenance
+);

go/ql/lib/semmle/go/dataflow/internal/FlowSummaryImplSpecific.qll

@@ -65,11 +65,9 @@ DataFlowType getSyntheticGlobalType(SummaryComponent::SyntheticGlobal sg) { any(
 predicate summaryElement(
   SummarizedCallableBase c, string input, string output, string kind, string provenance
 ) {
-  exists(
-    string namespace, string type, boolean subtypes, string name, string signature, string ext
-  |
-    summaryModel(namespace, type, subtypes, name, signature, ext, input, output, kind, provenance) and
-    c.asFunction() = interpretElement(namespace, type, subtypes, name, signature, ext).asEntity()
+  exists(string package, string type, boolean subtypes, string name, string signature, string ext |
+    summaryModel(package, type, subtypes, name, signature, ext, input, output, kind, provenance) and
+    c.asFunction() = interpretElement(package, type, subtypes, name, signature, ext).asEntity()
   )
 }
 
@@ -154,11 +152,9 @@ class SourceOrSinkElement extends TSourceOrSinkElement {
  * `output`, kind `kind`, and provenance `provenance`.
  */
 predicate sourceElement(SourceOrSinkElement e, string output, string kind, string provenance) {
-  exists(
-    string namespace, string type, boolean subtypes, string name, string signature, string ext
-  |
-    sourceModel(namespace, type, subtypes, name, signature, ext, output, kind, provenance) and
-    e = interpretElement(namespace, type, subtypes, name, signature, ext)
+  exists(string package, string type, boolean subtypes, string name, string signature, string ext |
+    sourceModel(package, type, subtypes, name, signature, ext, output, kind, provenance) and
+    e = interpretElement(package, type, subtypes, name, signature, ext)
   )
 }
 
@@ -167,11 +163,9 @@ predicate sourceElement(SourceOrSinkElement e, string output, string kind, strin
  * `input`, kind `kind` and provenance `provenance`.
  */
 predicate sinkElement(SourceOrSinkElement e, string input, string kind, string provenance) {
-  exists(
-    string namespace, string type, boolean subtypes, string name, string signature, string ext
-  |
-    sinkModel(namespace, type, subtypes, name, signature, ext, input, kind, provenance) and
-    e = interpretElement(namespace, type, subtypes, name, signature, ext)
+  exists(string package, string type, boolean subtypes, string name, string signature, string ext |
+    sinkModel(package, type, subtypes, name, signature, ext, input, kind, provenance) and
+    e = interpretElement(package, type, subtypes, name, signature, ext)
   )
 }
 

go/ql/lib/semmle/go/frameworks/stdlib/ArchiveTar.qll

@@ -3,13 +3,6 @@
  */
 
 import go
-private import semmle.go.dataflow.ExternalFlow
-
-private class FlowSummaries extends SummaryModelCsv {
-  override predicate row(string row) {
-    row = ["archive/tar;;true;FileInfoHeader;;;Argument[0];ReturnValue[0];taint"]
-  }
-}
 
 /** Provides models of commonly used functions in the `archive/tar` package. */
 module ArchiveTar {

go/ql/lib/semmle/go/frameworks/stdlib/NetHttp.qll

@@ -3,23 +3,6 @@
  */
 
 import go
-private import semmle.go.dataflow.ExternalFlow
-
-private class FlowSources extends SourceModelCsv {
-  override predicate row(string row) {
-    row =
-      [
-        "net/http;Request;true;Cookie;;;ReturnValue[0];remote",
-        "net/http;Request;true;Cookies;;;ReturnValue.ArrayElement;remote",
-        "net/http;Request;true;FormFile;;;ReturnValue[0..1];remote",
-        "net/http;Request;true;FormValue;;;ReturnValue;remote",
-        "net/http;Request;true;MultipartReader;;;ReturnValue[0];remote",
-        "net/http;Request;true;PostFormValue;;;ReturnValue;remote",
-        "net/http;Request;true;Referer;;;ReturnValue;remote",
-        "net/http;Request;true;UserAgent;;;ReturnValue;remote"
-      ]
-  }
-}
 
 /** Provides models of commonly used functions in the `net/http` package. */
 module NetHttp {

go/ql/lib/semmle/go/security/FlowSources.qll

@@ -23,7 +23,12 @@ module UntrustedFlowSource {
    */
   abstract class Range extends DataFlow::Node { }
 
-  class CsvRemoteSource extends Range {
-    CsvRemoteSource() { ExternalFlow::sourceNode(this, "remote") }
+  /**
+   * A source of data that is controlled by an untrusted user.
+   */
+  class MaDRemoteSource extends Range {
+    MaDRemoteSource() { ExternalFlow::sourceNode(this, "remote") }
   }
+
+  deprecated class CsvRemoteSource = MaDRemoteSource;
 }

go/ql/test/library-tests/semmle/go/dataflow/ExternalFlow/completetest.ext.yml

@@ -0,0 +1,38 @@
+extensions:
+  - addsTo:
+      pack: codeql/go-all
+      extensible: summaryModel
+    data:
+      - ["github.com/nonexistent/test", "T", False, "StepArgRes", "", "", "Argument[0]", "ReturnValue", "taint", "manual"]
+      - ["github.com/nonexistent/test", "T", False, "StepArgRes1", "", "", "Argument[0]", "ReturnValue[1]", "taint", "manual"]
+      - ["github.com/nonexistent/test", "T", False, "StepArgArg", "", "", "Argument[0]", "Argument[1]", "taint", "manual"]
+      - ["github.com/nonexistent/test", "T", False, "StepArgQual", "", "", "Argument[0]", "Argument[-1]", "taint", "manual"]
+      - ["github.com/nonexistent/test", "T", False, "StepQualRes", "", "", "Argument[-1]", "ReturnValue", "taint", "manual"]
+      - ["github.com/nonexistent/test", "T", False, "StepQualArg", "", "", "Argument[-1]", "Argument[0]", "taint", "manual"]
+      - ["github.com/nonexistent/test", "", False, "StepArgResNoQual", "", "", "Argument[0]", "ReturnValue", "taint", "manual"]
+      - ["github.com/nonexistent/test", "", False, "StepArgResArrayContent", "", "", "Argument[0]", "ReturnValue.ArrayElement", "taint", "manual"]
+      - ["github.com/nonexistent/test", "", False, "StepArgArrayContentRes", "", "", "Argument[0].ArrayElement", "ReturnValue", "taint", "manual"]
+      - ["github.com/nonexistent/test", "", False, "StepArgResCollectionContent", "", "", "Argument[0]", "ReturnValue.Element", "taint", "manual"]
+      - ["github.com/nonexistent/test", "", False, "StepArgCollectionContentRes", "", "", "Argument[0].Element", "ReturnValue", "taint", "manual"]
+      - ["github.com/nonexistent/test", "", False, "StepArgResMapKeyContent", "", "", "Argument[0]", "ReturnValue.MapKey", "taint", "manual"]
+      - ["github.com/nonexistent/test", "", False, "StepArgMapKeyContentRes", "", "", "Argument[0].MapKey", "ReturnValue", "taint", "manual"]
+      - ["github.com/nonexistent/test", "", False, "StepArgResMapValueContent", "", "", "Argument[0]", "ReturnValue.MapValue", "taint", "manual"]
+      - ["github.com/nonexistent/test", "", False, "StepArgMapValueContentRes", "", "", "Argument[0].MapValue", "ReturnValue", "taint", "manual"]
+      - ["github.com/nonexistent/test", "", False, "GetElement", "", "", "Argument[0].Element", "ReturnValue", "value", "manual"]
+      - ["github.com/nonexistent/test", "", False, "GetMapKey", "", "", "Argument[0].MapKey", "ReturnValue", "value", "manual"]
+      - ["github.com/nonexistent/test", "", False, "SetElement", "", "", "Argument[0]", "ReturnValue.Element", "value", "manual"]
+      - ["github.com/nonexistent/test", "C", False, "Get", "", "", "Argument[-1].Field[github.com/nonexistent/test.C.F]", "ReturnValue", "value", "manual"]
+      - ["github.com/nonexistent/test", "C", False, "Set", "", "", "Argument[0]", "Argument[-1].Field[github.com/nonexistent/test.C.F]", "value", "manual"]
+
+  - addsTo:
+      pack: codeql/go-all
+      extensible: sourceModel
+    data:
+      - ["github.com/nonexistent/test", "A", False, "Src1", "", "", "ReturnValue", "qltest", "manual"]
+
+  - addsTo:
+      pack: codeql/go-all
+      extensible: sinkModel
+    data:
+      - ["github.com/nonexistent/test", "B", False, "Sink1", "", "", "Argument[0]", "qltest", "manual"]
+      - ["github.com/nonexistent/test", "B", False, "SinkManyArgs", "", "", "Argument[0..2]", "qltest", "manual"]

go/ql/test/library-tests/semmle/go/dataflow/ExternalFlow/completetest.ql

@@ -4,60 +4,10 @@
 
 import go
 import semmle.go.dataflow.ExternalFlow
-import CsvValidation
+import ModelValidation
 import semmle.go.dataflow.internal.FlowSummaryImpl as FlowSummaryImpl
 import TestUtilities.InlineFlowTest
 
-class SummaryModelTest extends SummaryModelCsv {
-  override predicate row(string row) {
-    row =
-      [
-        //`namespace; type; subtypes; name; signature; ext; input; output; kind`
-        "github.com/nonexistent/test;T;false;StepArgRes;;;Argument[0];ReturnValue;taint",
-        "github.com/nonexistent/test;T;false;StepArgRes1;;;Argument[0];ReturnValue[1];taint",
-        "github.com/nonexistent/test;T;false;StepArgArg;;;Argument[0];Argument[1];taint",
-        "github.com/nonexistent/test;T;false;StepArgQual;;;Argument[0];Argument[-1];taint",
-        "github.com/nonexistent/test;T;false;StepQualRes;;;Argument[-1];ReturnValue;taint",
-        "github.com/nonexistent/test;T;false;StepQualArg;;;Argument[-1];Argument[0];taint",
-        "github.com/nonexistent/test;;false;StepArgResNoQual;;;Argument[0];ReturnValue;taint",
-        "github.com/nonexistent/test;;false;StepArgResArrayContent;;;Argument[0];ReturnValue.ArrayElement;taint",
-        "github.com/nonexistent/test;;false;StepArgArrayContentRes;;;Argument[0].ArrayElement;ReturnValue;taint",
-        "github.com/nonexistent/test;;false;StepArgResCollectionContent;;;Argument[0];ReturnValue.Element;taint",
-        "github.com/nonexistent/test;;false;StepArgCollectionContentRes;;;Argument[0].Element;ReturnValue;taint",
-        "github.com/nonexistent/test;;false;StepArgResMapKeyContent;;;Argument[0];ReturnValue.MapKey;taint",
-        "github.com/nonexistent/test;;false;StepArgMapKeyContentRes;;;Argument[0].MapKey;ReturnValue;taint",
-        "github.com/nonexistent/test;;false;StepArgResMapValueContent;;;Argument[0];ReturnValue.MapValue;taint",
-        "github.com/nonexistent/test;;false;StepArgMapValueContentRes;;;Argument[0].MapValue;ReturnValue;taint",
-        "github.com/nonexistent/test;;false;GetElement;;;Argument[0].Element;ReturnValue;value",
-        "github.com/nonexistent/test;;false;GetMapKey;;;Argument[0].MapKey;ReturnValue;value",
-        "github.com/nonexistent/test;;false;SetElement;;;Argument[0];ReturnValue.Element;value",
-        "github.com/nonexistent/test;C;false;Get;;;Argument[-1].Field[github.com/nonexistent/test.C.F];ReturnValue;value",
-        "github.com/nonexistent/test;C;false;Set;;;Argument[0];Argument[-1].Field[github.com/nonexistent/test.C.F];value",
-      ]
-  }
-}
-
-class SourceModelTest extends SourceModelCsv {
-  override predicate row(string row) {
-    row =
-      [
-        //`namespace; type; subtypes; name; -; ext; output; kind`
-        "github.com/nonexistent/test;A;false;Src1;;;ReturnValue;qltest"
-      ]
-  }
-}
-
-class SinkModelTest extends SinkModelCsv {
-  override predicate row(string row) {
-    row =
-      [
-        //`namespace; type; subtypes; name; -; ext; input; kind`
-        "github.com/nonexistent/test;B;false;Sink1;;;Argument[0];qltest",
-        "github.com/nonexistent/test;B;false;SinkManyArgs;;;Argument[0..2];qltest"
-      ]
-  }
-}
-
 class Config extends TaintTracking::Configuration {
   Config() { this = "external-flow-test" }
 

go/ql/test/library-tests/semmle/go/dataflow/ExternalFlow/sinks.ext.yml

@@ -0,0 +1,8 @@
+extensions:
+  - addsTo:
+      pack: codeql/go-all
+      extensible: sinkModel
+    data:
+      - ["github.com/nonexistent/test", "B", False, "Sink1", "", "", "Argument[0]", "qltest", "manual"]
+      - ["github.com/nonexistent/test", "B", False, "SinkMethod", "", "", "Argument[-1]", "qltest", "manual"]
+      - ["github.com/nonexistent/test", "B", False, "SinkManyArgs", "", "", "Argument[0..2]", "qltest", "manual"]

go/ql/test/library-tests/semmle/go/dataflow/ExternalFlow/sinks.ql

@@ -1,18 +1,6 @@
 import go
 import semmle.go.dataflow.ExternalFlow
-import CsvValidation
-
-class SinkModelTest extends SinkModelCsv {
-  override predicate row(string row) {
-    row =
-      [
-        //`namespace; type; subtypes; name; -; ext; input; kind`
-        "github.com/nonexistent/test;B;false;Sink1;;;Argument[0];qltest",
-        "github.com/nonexistent/test;B;false;SinkMethod;;;Argument[-1];qltest",
-        "github.com/nonexistent/test;B;false;SinkManyArgs;;;Argument[0..2];qltest",
-      ]
-  }
-}
+import ModelValidation
 
 from DataFlow::Node node, string kind
 where sinkNode(node, kind)

go/ql/test/library-tests/semmle/go/dataflow/ExternalFlow/srcs.ext.yml

@@ -0,0 +1,11 @@
+extensions:
+  - addsTo:
+      pack: codeql/go-all
+      extensible: sourceModel
+    data:
+      - ["github.com/nonexistent/test", "A", False, "Src1", "", "", "ReturnValue", "qltest", "manual"]
+      - ["github.com/nonexistent/test", "A", False, "Src2", "", "", "ReturnValue", "qltest", "manual"]
+      - ["github.com/nonexistent/test", "A", True, "Src2", "", "", "ReturnValue", "qltest-w-subtypes", "manual"]
+      - ["github.com/nonexistent/test", "A", False, "SrcArg", "", "", "Argument[0]", "qltest-arg", "manual"]
+      - ["github.com/nonexistent/test", "A", False, "Src3", "", "", "ReturnValue[0]", "qltest", "manual"]
+      - ["github.com/nonexistent/test", "A", True, "Src3", "", "", "ReturnValue[1]", "qltest-w-subtypes", "manual"]

go/ql/test/library-tests/semmle/go/dataflow/ExternalFlow/srcs.ql

@@ -1,21 +1,6 @@
 import go
 import semmle.go.dataflow.ExternalFlow
-import CsvValidation
-
-class SourceModelTest extends SourceModelCsv {
-  override predicate row(string row) {
-    row =
-      [
-        //`namespace; type; subtypes; name; -; ext; output; kind`
-        "github.com/nonexistent/test;A;false;Src1;;;ReturnValue;qltest",
-        "github.com/nonexistent/test;A;false;Src2;;;ReturnValue;qltest",
-        "github.com/nonexistent/test;A;true;Src2;;;ReturnValue;qltest-w-subtypes",
-        "github.com/nonexistent/test;A;false;SrcArg;;;Argument[0];qltest-arg",
-        "github.com/nonexistent/test;A;false;Src3;;;ReturnValue[0];qltest",
-        "github.com/nonexistent/test;A;true;Src3;;;ReturnValue[1];qltest-w-subtypes"
-      ]
-  }
-}
+import ModelValidation
 
 from DataFlow::Node node, string kind
 where sourceNode(node, kind)

go/ql/test/library-tests/semmle/go/dataflow/ExternalFlow/steps.ext.yml

@@ -0,0 +1,14 @@
+extensions:
+  - addsTo:
+      pack: codeql/go-all
+      extensible: summaryModel
+    data:
+      - ["github.com/nonexistent/test", "T", False, "StepArgRes", "", "", "Argument[0]", "ReturnValue", "taint", "manual"]
+      - ["github.com/nonexistent/test", "T", False, "StepArgRes1", "", "", "Argument[0]", "ReturnValue[1]", "taint", "manual"]
+      - ["github.com/nonexistent/test", "T", False, "StepArgArg", "", "", "Argument[0]", "Argument[1]", "taint", "manual"]
+      - ["github.com/nonexistent/test", "T", False, "StepArgQual", "", "", "Argument[0]", "Argument[-1]", "taint", "manual"]
+      - ["github.com/nonexistent/test", "T", False, "StepQualRes", "", "", "Argument[-1]", "ReturnValue", "taint", "manual"]
+      - ["github.com/nonexistent/test", "T", False, "StepQualArg", "", "", "Argument[-1]", "Argument[0]", "taint", "manual"]
+      - ["github.com/nonexistent/test", "", False, "StepArgResNoQual", "", "", "Argument[0]", "ReturnValue", "taint", "manual"]
+      - ["github.com/nonexistent/test", "", False, "StepArgResContent", "", "", "Argument[0]", "ReturnValue.ArrayElement", "taint", "manual"]
+      - ["github.com/nonexistent/test", "", False, "StepArgContentRes", "", "", "Argument[0].ArrayElement", "ReturnValue", "taint", "manual"]

go/ql/test/library-tests/semmle/go/dataflow/ExternalFlow/steps.ql

@@ -1,26 +1,8 @@
 import go
 import semmle.go.dataflow.ExternalFlow
-import CsvValidation
+import ModelValidation
 import semmle.go.dataflow.internal.FlowSummaryImpl as FlowSummaryImpl
 
-class SummaryModelTest extends SummaryModelCsv {
-  override predicate row(string row) {
-    row =
-      [
-        //`namespace; type; subtypes; name; -; ext; input; output; kind`
-        "github.com/nonexistent/test;T;false;StepArgRes;;;Argument[0];ReturnValue;taint",
-        "github.com/nonexistent/test;T;false;StepArgRes1;;;Argument[0];ReturnValue[1];taint",
-        "github.com/nonexistent/test;T;false;StepArgArg;;;Argument[0];Argument[1];taint",
-        "github.com/nonexistent/test;T;false;StepArgQual;;;Argument[0];Argument[-1];taint",
-        "github.com/nonexistent/test;T;false;StepQualRes;;;Argument[-1];ReturnValue;taint",
-        "github.com/nonexistent/test;T;false;StepQualArg;;;Argument[-1];Argument[0];taint",
-        "github.com/nonexistent/test;;false;StepArgResNoQual;;;Argument[0];ReturnValue;taint",
-        "github.com/nonexistent/test;;false;StepArgResContent;;;Argument[0];ReturnValue.ArrayElement;taint",
-        "github.com/nonexistent/test;;false;StepArgContentRes;;;Argument[0].ArrayElement;ReturnValue;taint"
-      ]
-  }
-}
-
 from DataFlow::Node node1, DataFlow::Node node2
 where FlowSummaryImpl::Private::Steps::summaryThroughStepTaint(node1, node2, _)
 select node1, node2

go/ql/test/library-tests/semmle/go/dataflow/ExternalFlowVarArgs/Flows.ext.yml

@@ -0,0 +1,10 @@
+extensions:
+  - addsTo:
+      pack: codeql/go-all
+      extensible: summaryModel
+    data:
+      - ["github.com/nonexistent/test", "", False, "FunctionWithParameter", "", "", "Argument[0]", "ReturnValue", "value", "manual"]
+      - ["github.com/nonexistent/test", "", False, "FunctionWithSliceParameter", "", "", "Argument[0].ArrayElement", "ReturnValue", "value", "manual"]
+      - ["github.com/nonexistent/test", "", False, "FunctionWithVarArgsParameter", "", "", "Argument[0].ArrayElement", "ReturnValue", "value", "manual"]
+      - ["github.com/nonexistent/test", "", False, "FunctionWithSliceOfStructsParameter", "", "", "Argument[0].ArrayElement.Field[github.com/nonexistent/test.A.Field]", "ReturnValue", "value", "manual"]
+      - ["github.com/nonexistent/test", "", False, "FunctionWithVarArgsOfStructsParameter", "", "", "Argument[0].ArrayElement.Field[github.com/nonexistent/test.A.Field]", "ReturnValue", "value", "manual"]

go/ql/test/library-tests/semmle/go/dataflow/ExternalFlowVarArgs/Flows.ql

@@ -1,22 +1,8 @@
 import go
 import semmle.go.dataflow.ExternalFlow
-import CsvValidation
+import ModelValidation
 import TestUtilities.InlineExpectationsTest
 
-class SummaryModelTest extends SummaryModelCsv {
-  override predicate row(string row) {
-    row =
-      [
-        //`namespace; type; subtypes; name; signature; ext; input; output; kind`
-        "github.com/nonexistent/test;;false;FunctionWithParameter;;;Argument[0];ReturnValue;value",
-        "github.com/nonexistent/test;;false;FunctionWithSliceParameter;;;Argument[0].ArrayElement;ReturnValue;value",
-        "github.com/nonexistent/test;;false;FunctionWithVarArgsParameter;;;Argument[0].ArrayElement;ReturnValue;value",
-        "github.com/nonexistent/test;;false;FunctionWithSliceOfStructsParameter;;;Argument[0].ArrayElement.Field[github.com/nonexistent/test.A.Field];ReturnValue;value",
-        "github.com/nonexistent/test;;false;FunctionWithVarArgsOfStructsParameter;;;Argument[0].ArrayElement.Field[github.com/nonexistent/test.A.Field];ReturnValue;value"
-      ]
-  }
-}
-
 class DataConfiguration extends DataFlow::Configuration {
   DataConfiguration() { this = "data-configuration" }
 

go/ql/test/library-tests/semmle/go/dataflow/VarArgsWithFunctionModels/Flows.ql

@@ -1,6 +1,6 @@
 import go
 import semmle.go.dataflow.ExternalFlow
-import CsvValidation
+import ModelValidation
 // import DataFlow::PartialPathGraph
 import TestUtilities.InlineExpectationsTest