Merge branch 'main' into gradio-model

python/ql/lib/semmle/python/frameworks/Asyncpg.model.yml

@@ -0,0 +1,29 @@
+extensions:
+  - addsTo:
+      pack: codeql/python-all
+      extensible: sinkModel
+    data:
+      # `Connection`s and `ConnectionPool`s provide some methods that execute SQL.
+      - ['asyncpg.~Connection', 'Member[copy_from_query,execute,fetch,fetchrow,fetchval].Argument[0,query:]', 'sql-injection']
+      - ['asyncpg.~Connection', 'Member[executemany].Argument[0,command:]', 'sql-injection']
+      # A model of `Connection` and `ConnectionPool`, which provide some methods that access the file system.
+      - ['asyncpg.~Connection', 'Member[copy_from_query,copy_from_table].Argument[output:]', 'path-injection']
+      - ['asyncpg.~Connection', 'Member[copy_to_table].Argument[source:]', 'path-injection']
+      # the `PreparedStatement` class in `asyncpg`.
+      - ['asyncpg.Connection', 'Member[prepare].Argument[0,query:]', 'sql-injection']
+
+  - addsTo:
+      pack: codeql/python-all
+      extensible: typeModel
+    data:
+      # a `ConnectionPool` that is created when the result of `asyncpg.create_pool()` is awaited.
+      - ['asyncpg.Connection', 'asyncpg.ConnectionPool', 'Member[acquire].ReturnValue.Awaited']
+      # a `Connection` that is created when
+      # * - the result of `asyncpg.connect()` is awaited.
+      # * - the result of calling `acquire` on a `ConnectionPool` is awaited.
+      - ['asyncpg.Connection', 'asyncpg', 'Member[connect].ReturnValue.Awaited']
+      - ['asyncpg.Connection', 'asyncpg', 'Member[connection].Member[connect].ReturnValue.Awaited']
+      - ['asyncpg.ConnectionPool', 'asyncpg', 'Member[create_pool].ReturnValue.Awaited']
+      # Creating an internal `~Connection` type that contains both `Connection` and `ConnectionPool`.
+      - ['asyncpg.~Connection', 'asyncpg.Connection', '']
+      - ['asyncpg.~Connection', 'asyncpg.ConnectionPool', '']

python/ql/lib/semmle/python/frameworks/Asyncpg.qll

@@ -11,43 +11,6 @@ private import semmle.python.frameworks.data.ModelsAsData
 
 /** Provides models for the `asyncpg` PyPI package. */
 private module Asyncpg {
-  class AsyncpgModel extends ModelInput::TypeModelCsv {
-    override predicate row(string row) {
-      // type1;type2;path
-      row =
-        [
-          // a `ConnectionPool` that is created when the result of `asyncpg.create_pool()` is awaited.
-          "asyncpg.ConnectionPool;asyncpg;Member[create_pool].ReturnValue.Awaited",
-          // a `Connection` that is created when
-          // * - the result of `asyncpg.connect()` is awaited.
-          // * - the result of calling `acquire` on a `ConnectionPool` is awaited.
-          "asyncpg.Connection;asyncpg;Member[connect].ReturnValue.Awaited",
-          "asyncpg.Connection;asyncpg;Member[connection].Member[connect].ReturnValue.Awaited",
-          "asyncpg.Connection;asyncpg.ConnectionPool;Member[acquire].ReturnValue.Awaited",
-          // Creating an internal `~Connection` type that contains both `Connection` and `ConnectionPool`.
-          "asyncpg.~Connection;asyncpg.Connection;", //
-          "asyncpg.~Connection;asyncpg.ConnectionPool;"
-        ]
-    }
-  }
-
-  class AsyncpgSink extends ModelInput::SinkModelCsv {
-    // type;path;kind
-    override predicate row(string row) {
-      row =
-        [
-          // `Connection`s and `ConnectionPool`s provide some methods that execute SQL.
-          "asyncpg.~Connection;Member[copy_from_query,execute,fetch,fetchrow,fetchval].Argument[0,query:];sql-injection",
-          "asyncpg.~Connection;Member[executemany].Argument[0,command:];sql-injection",
-          // A model of `Connection` and `ConnectionPool`, which provide some methods that access the file system.
-          "asyncpg.~Connection;Member[copy_from_query,copy_from_table].Argument[output:];path-injection",
-          "asyncpg.~Connection;Member[copy_to_table].Argument[source:];path-injection",
-          // the `PreparedStatement` class in `asyncpg`.
-          "asyncpg.Connection;Member[prepare].Argument[0,query:];sql-injection",
-        ]
-    }
-  }
-
   /**
    * Provides models of the `Cursor` class in `asyncpg`.
    * `Cursor`s are created

python/ql/lib/semmle/python/frameworks/Cryptodome.qll

@@ -83,7 +83,7 @@ private module CryptodomeModel {
 
     /** Gets the name of the curve to use, as well as the origin that explains how we obtained this name. */
     string getCurveWithOrigin(DataFlow::Node origin) {
-      exists(StrConst str | origin = DataFlow::exprNode(str) |
+      exists(StringLiteral str | origin = DataFlow::exprNode(str) |
         origin = this.getCurveArg().getALocalSource() and
         result = str.getText()
       )

python/ql/lib/semmle/python/frameworks/Django.qll

@@ -2862,14 +2862,14 @@ module PrivateDjango {
         //
         // This also strongly implies that `mw` is in fact a Django middleware setting and
         // not just a variable named `MIDDLEWARE`.
-        list.getAnElt().(StrConst).getText() =
+        list.getAnElt().(StringLiteral).getText() =
           "django.contrib.auth.middleware.AuthenticationMiddleware"
       )
     }
 
     override boolean getVerificationSetting() {
       if
-        list.getAnElt().(StrConst).getText() in [
+        list.getAnElt().(StringLiteral).getText() in [
             "django.middleware.csrf.CsrfViewMiddleware",
             // see https://github.com/mozilla/django-session-csrf
             "session_csrf.CsrfMiddleware"

python/ql/lib/semmle/python/frameworks/FastApi.qll

@@ -183,7 +183,7 @@ module FastApi {
       |
         exists(Assign assign | assign = cls.getAStmt() |
           assign.getATarget().(Name).getId() = "media_type" and
-          result = assign.getValue().(StrConst).getText()
+          result = assign.getValue().(StringLiteral).getText()
         )
         or
         // TODO: this should use a proper MRO calculation instead
@@ -372,7 +372,7 @@ module FastApi {
           headers.accesses(instance(), "headers") and
           this.calls(headers, "append") and
           keyArg in [this.getArg(0), this.getArgByName("key")] and
-          keyArg.getALocalSource().asExpr().(StrConst).getText().toLowerCase() = "set-cookie"
+          keyArg.getALocalSource().asExpr().(StringLiteral).getText().toLowerCase() = "set-cookie"
         )
       }
 

python/ql/lib/semmle/python/frameworks/Rsa.qll

@@ -80,7 +80,7 @@ private module Rsa {
       result.getName() = "RSA"
       or
       // hashing part
-      exists(StrConst str, DataFlow::Node hashNameArg |
+      exists(StringLiteral str, DataFlow::Node hashNameArg |
         hashNameArg in [this.getArg(2), this.getArgByName("hash_method")] and
         DataFlow::exprNode(str) = hashNameArg.getALocalSource() and
         result.matchesName(str.getText())
@@ -132,7 +132,7 @@ private module Rsa {
     override DataFlow::Node getInitialization() { result = this }
 
     override Cryptography::CryptographicAlgorithm getAlgorithm() {
-      exists(StrConst str, DataFlow::Node hashNameArg |
+      exists(StringLiteral str, DataFlow::Node hashNameArg |
         hashNameArg in [this.getArg(1), this.getArgByName("method_name")] and
         DataFlow::exprNode(str) = hashNameArg.getALocalSource() and
         result.matchesName(str.getText())

python/ql/lib/semmle/python/frameworks/ServerLess.qll

@@ -8,7 +8,7 @@
  */
 
 import python
-import codeql.serverless.ServerLess
+private import codeql.serverless.ServerLess
 import semmle.python.dataflow.new.DataFlow
 import semmle.python.dataflow.new.RemoteFlowSources
 

python/ql/lib/semmle/python/frameworks/Stdlib.qll

@@ -2785,7 +2785,7 @@ module StdlibPrivate {
   /** Gets a call to `hashlib.new` with `algorithmName` as the first argument. */
   private API::CallNode hashlibNewCall(string algorithmName) {
     algorithmName =
-      result.getParameter(0, "name").getAValueReachingSink().asExpr().(StrConst).getText() and
+      result.getParameter(0, "name").getAValueReachingSink().asExpr().(StringLiteral).getText() and
     result = API::moduleImport("hashlib").getMember("new").getACall()
   }
 
@@ -2908,7 +2908,8 @@ module StdlibPrivate {
       exists(string algorithmName | result.matchesName(algorithmName) |
         this.getDigestArg().asSink() = hashlibMember(algorithmName).asSource()
         or
-        this.getDigestArg().getAValueReachingSink().asExpr().(StrConst).getText() = algorithmName
+        this.getDigestArg().getAValueReachingSink().asExpr().(StringLiteral).getText() =
+          algorithmName
       )
     }
 
@@ -4418,7 +4419,7 @@ module StdlibPrivate {
 
     override DataFlow::CallCfgNode getACall() {
       result.(DataFlow::MethodCallNode).getMethodName() = "pop" and
-      result.getArg(0).getALocalSource().asExpr().(StrConst).getText() = key
+      result.getArg(0).getALocalSource().asExpr().(StringLiteral).getText() = key
     }
 
     override DataFlow::ArgumentNode getACallback() { none() }
@@ -4441,7 +4442,7 @@ module StdlibPrivate {
 
     override DataFlow::CallCfgNode getACall() {
       result.(DataFlow::MethodCallNode).getMethodName() = "get" and
-      result.getArg(0).getALocalSource().asExpr().(StrConst).getText() = key
+      result.getArg(0).getALocalSource().asExpr().(StringLiteral).getText() = key
     }
 
     override DataFlow::ArgumentNode getACallback() { none() }
@@ -4541,7 +4542,7 @@ module StdlibPrivate {
 
     override DataFlow::CallCfgNode getACall() {
       result.(DataFlow::MethodCallNode).getMethodName() = "setdefault" and
-      result.getArg(0).getALocalSource().asExpr().(StrConst).getText() = key
+      result.getArg(0).getALocalSource().asExpr().(StringLiteral).getText() = key
     }
 
     override DataFlow::ArgumentNode getACallback() { none() }

python/ql/lib/semmle/python/frameworks/Urllib3.qll

@@ -78,7 +78,7 @@ module Urllib3 {
           // see https://urllib3.readthedocs.io/en/stable/user-guide.html?highlight=cert_reqs#certificate-verification
           disablingNode = constructor.getKeywordParameter("cert_reqs").asSink() and
           argumentOrigin = constructor.getKeywordParameter("cert_reqs").getAValueReachingSink() and
-          argumentOrigin.asExpr().(StrConst).getText() = "CERT_NONE"
+          argumentOrigin.asExpr().(StringLiteral).getText() = "CERT_NONE"
           or
           // assert_hostname
           // see https://urllib3.readthedocs.io/en/stable/reference/urllib3.connectionpool.html?highlight=assert_hostname#urllib3.HTTPSConnectionPool

python/ql/lib/semmle/python/frameworks/data/ModelsAsData.qll

@@ -22,7 +22,7 @@ private import semmle.python.dataflow.new.FlowSummary
 /**
  * A remote flow source originating from a CSV source row.
  */
-private class RemoteFlowSourceFromCsv extends RemoteFlowSource {
+private class RemoteFlowSourceFromCsv extends RemoteFlowSource::Range {
   RemoteFlowSourceFromCsv() { this = ModelOutput::getASourceNode("remote").asSource() }
 
   override string getSourceType() { result = "Remote flow (from model)" }
@@ -33,7 +33,7 @@ private class SummarizedCallableFromModel extends SummarizedCallable {
   string path;
 
   SummarizedCallableFromModel() {
-    ModelOutput::relevantSummaryModel(type, path, _, _, _) and
+    ModelOutput::relevantSummaryModel(type, path, _, _, _, _) and
     this = type + ";" + path
   }
 
@@ -46,8 +46,10 @@ private class SummarizedCallableFromModel extends SummarizedCallable {
     )
   }
 
-  override predicate propagatesFlow(string input, string output, boolean preservesValue) {
-    exists(string kind | ModelOutput::relevantSummaryModel(type, path, input, output, kind) |
+  override predicate propagatesFlow(
+    string input, string output, boolean preservesValue, string model
+  ) {
+    exists(string kind | ModelOutput::relevantSummaryModel(type, path, input, output, kind, model) |
       kind = "value" and
       preservesValue = true
       or

python/ql/lib/semmle/python/frameworks/data/internal/ApiGraphModels.qll

@@ -1,17 +1,17 @@
 /**
  * INTERNAL use only. This is an experimental API subject to change without notice.
  *
- * Provides classes and predicates for dealing with flow models specified in CSV format.
+ * Provides classes and predicates for dealing with flow models specified in extensible predicates.
  *
- * The CSV specification has the following columns:
+ * The extensible predicates have the following columns:
  * - Sources:
- *   `type; path; kind`
+ *   `type, path, kind`
  * - Sinks:
- *   `type; path; kind`
+ *   `type, path, kind`
  * - Summaries:
- *   `type; path; input; output; kind`
+ *   `type, path, input, output, kind`
  * - Types:
- *   `type1; type2; path`
+ *   `type1, type2, path`
  *
  * The interpretation of a row is similar to API-graphs with a left-to-right
  * reading.
@@ -76,11 +76,13 @@ private import codeql.dataflow.internal.AccessPathSyntax
 /** Module containing hooks for providing input data to be interpreted as a model. */
 module ModelInput {
   /**
+   * DEPRECATED: Use the extensible predicate `sourceModel` instead.
+   *
    * A unit class for adding additional source model rows.
    *
    * Extend this class to add additional source definitions.
    */
-  class SourceModelCsv extends Unit {
+  deprecated class SourceModelCsv extends Unit {
     /**
      * Holds if `row` specifies a source definition.
      *
@@ -93,15 +95,17 @@ module ModelInput {
      *
      * The kind `remote` represents a general remote flow source.
      */
-    abstract predicate row(string row);
+    abstract deprecated predicate row(string row);
   }
 
   /**
    * A unit class for adding additional sink model rows.
    *
    * Extend this class to add additional sink definitions.
+   *
+   * DEPRECATED: Use the extensible predicate `sinkModel` instead.
    */
-  class SinkModelCsv extends Unit {
+  deprecated class SinkModelCsv extends Unit {
     /**
      * Holds if `row` specifies a sink definition.
      *
@@ -112,15 +116,17 @@ module ModelInput {
      * indicates that the value at `(type, path)` should be seen as a sink
      * of the given `kind`.
      */
-    abstract predicate row(string row);
+    abstract deprecated predicate row(string row);
   }
 
   /**
    * A unit class for adding additional summary model rows.
    *
    * Extend this class to add additional flow summary definitions.
+   *
+   * DEPRECATED: Use the extensible predicate `summaryModel` instead.
    */
-  class SummaryModelCsv extends Unit {
+  deprecated class SummaryModelCsv extends Unit {
     /**
      * Holds if `row` specifies a summary definition.
      *
@@ -134,15 +140,18 @@ module ModelInput {
      * `kind` should be either `value` or `taint`, for value-preserving or taint-preserving steps,
      * respectively.
      */
-    abstract predicate row(string row);
+    abstract deprecated predicate row(string row);
   }
 
   /**
    * A unit class for adding additional type model rows.
    *
    * Extend this class to add additional type definitions.
+   *
+   * DEPRECATED: Use the extensible predicate `typeModel` or the class
+   * `TypeModel` instead.
    */
-  class TypeModelCsv extends Unit {
+  deprecated class TypeModelCsv extends Unit {
     /**
      * Holds if `row` specifies a type definition.
      *
@@ -152,7 +161,7 @@ module ModelInput {
      * ```
      * indicates that `(type2, path)` should be seen as an instance of `type1`.
      */
-    abstract predicate row(string row);
+    abstract deprecated predicate row(string row);
   }
 
   /**
@@ -186,8 +195,10 @@ module ModelInput {
 
   /**
    * A unit class for adding additional type variable model rows.
+   *
+   * DEPRECATED: Use the extensible predicate `typeVariableModel` instead.
    */
-  class TypeVariableModelCsv extends Unit {
+  deprecated class TypeVariableModelCsv extends Unit {
     /**
      * Holds if `row` specifies a path through a type variable.
      *
@@ -197,7 +208,7 @@ module ModelInput {
      * ```
      * means `path` can be substituted for a token `TypeVar[name]`.
      */
-    abstract predicate row(string row);
+    abstract deprecated predicate row(string row);
   }
 }
 
@@ -216,87 +227,141 @@ abstract class TestAllModels extends Unit { }
  * does not preserve empty trailing substrings.
  */
 bindingset[result]
-private string inversePad(string s) { s = result + ";dummy" }
+deprecated private string inversePad(string s) { s = result + ";dummy" }
 
-private predicate sourceModel(string row) { any(SourceModelCsv s).row(inversePad(row)) }
+deprecated private predicate sourceModel(string row) { any(SourceModelCsv s).row(inversePad(row)) }
 
-private predicate sinkModel(string row) { any(SinkModelCsv s).row(inversePad(row)) }
+deprecated private predicate sinkModel(string row) { any(SinkModelCsv s).row(inversePad(row)) }
 
-private predicate summaryModel(string row) { any(SummaryModelCsv s).row(inversePad(row)) }
+deprecated private predicate summaryModel(string row) {
+  any(SummaryModelCsv s).row(inversePad(row))
+}
 
-private predicate typeModel(string row) { any(TypeModelCsv s).row(inversePad(row)) }
+deprecated private predicate typeModel(string row) { any(TypeModelCsv s).row(inversePad(row)) }
 
-private predicate typeVariableModel(string row) { any(TypeVariableModelCsv s).row(inversePad(row)) }
+deprecated private predicate typeVariableModel(string row) {
+  any(TypeVariableModelCsv s).row(inversePad(row))
+}
+
+private class DeprecationAdapter extends Unit {
+  abstract predicate sourceModel(string type, string path, string kind);
+
+  abstract predicate sinkModel(string type, string path, string kind);
+
+  abstract predicate summaryModel(string type, string path, string input, string output, string kind);
+
+  abstract predicate typeModel(string type1, string type2, string path);
+
+  abstract predicate typeVariableModel(string name, string path);
+}
+
+private class DeprecationAdapterImpl extends DeprecationAdapter {
+  deprecated override predicate sourceModel(string type, string path, string kind) {
+    exists(string row |
+      sourceModel(row) and
+      row.splitAt(";", 0) = type and
+      row.splitAt(";", 1) = path and
+      row.splitAt(";", 2) = kind
+    )
+  }
+
+  deprecated override predicate sinkModel(string type, string path, string kind) {
+    exists(string row |
+      sinkModel(row) and
+      row.splitAt(";", 0) = type and
+      row.splitAt(";", 1) = path and
+      row.splitAt(";", 2) = kind
+    )
+  }
+
+  deprecated override predicate summaryModel(
+    string type, string path, string input, string output, string kind
+  ) {
+    exists(string row |
+      summaryModel(row) and
+      row.splitAt(";", 0) = type and
+      row.splitAt(";", 1) = path and
+      row.splitAt(";", 2) = input and
+      row.splitAt(";", 3) = output and
+      row.splitAt(";", 4) = kind
+    )
+  }
+
+  deprecated override predicate typeModel(string type1, string type2, string path) {
+    exists(string row |
+      typeModel(row) and
+      row.splitAt(";", 0) = type1 and
+      row.splitAt(";", 1) = type2 and
+      row.splitAt(";", 2) = path
+    )
+  }
+
+  deprecated override predicate typeVariableModel(string name, string path) {
+    exists(string row |
+      typeVariableModel(row) and
+      row.splitAt(";", 0) = name and
+      row.splitAt(";", 1) = path
+    )
+  }
+}
 
 /** Holds if a source model exists for the given parameters. */
-predicate sourceModel(string type, string path, string kind) {
-  exists(string row |
-    sourceModel(row) and
-    row.splitAt(";", 0) = type and
-    row.splitAt(";", 1) = path and
-    row.splitAt(";", 2) = kind
-  )
+predicate sourceModel(string type, string path, string kind, string model) {
+  any(DeprecationAdapter a).sourceModel(type, path, kind) and
+  model = "SourceModelCsv"
   or
-  Extensions::sourceModel(type, path, kind)
+  exists(QlBuiltins::ExtensionId madId |
+    Extensions::sourceModel(type, path, kind, madId) and
+    model = "MaD:" + madId.toString()
+  )
 }
 
 /** Holds if a sink model exists for the given parameters. */
-private predicate sinkModel(string type, string path, string kind) {
-  exists(string row |
-    sinkModel(row) and
-    row.splitAt(";", 0) = type and
-    row.splitAt(";", 1) = path and
-    row.splitAt(";", 2) = kind
-  )
+private predicate sinkModel(string type, string path, string kind, string model) {
+  any(DeprecationAdapter a).sinkModel(type, path, kind) and
+  model = "SinkModelCsv"
   or
-  Extensions::sinkModel(type, path, kind)
+  exists(QlBuiltins::ExtensionId madId |
+    Extensions::sinkModel(type, path, kind, madId) and
+    model = "MaD:" + madId.toString()
+  )
 }
 
 /** Holds if a summary model `row` exists for the given parameters. */
-private predicate summaryModel(string type, string path, string input, string output, string kind) {
-  exists(string row |
-    summaryModel(row) and
-    row.splitAt(";", 0) = type and
-    row.splitAt(";", 1) = path and
-    row.splitAt(";", 2) = input and
-    row.splitAt(";", 3) = output and
-    row.splitAt(";", 4) = kind
-  )
+private predicate summaryModel(
+  string type, string path, string input, string output, string kind, string model
+) {
+  any(DeprecationAdapter a).summaryModel(type, path, input, output, kind) and
+  model = "SummaryModelCsv"
   or
-  Extensions::summaryModel(type, path, input, output, kind)
+  exists(QlBuiltins::ExtensionId madId |
+    Extensions::summaryModel(type, path, input, output, kind, madId) and
+    model = "MaD:" + madId.toString()
+  )
 }
 
 /** Holds if a type model exists for the given parameters. */
-private predicate typeModel(string type1, string type2, string path) {
-  exists(string row |
-    typeModel(row) and
-    row.splitAt(";", 0) = type1 and
-    row.splitAt(";", 1) = type2 and
-    row.splitAt(";", 2) = path
-  )
+predicate typeModel(string type1, string type2, string path) {
+  any(DeprecationAdapter a).typeModel(type1, type2, path)
   or
   Extensions::typeModel(type1, type2, path)
 }
 
 /** Holds if a type variable model exists for the given parameters. */
 private predicate typeVariableModel(string name, string path) {
-  exists(string row |
-    typeVariableModel(row) and
-    row.splitAt(";", 0) = name and
-    row.splitAt(";", 1) = path
-  )
+  any(DeprecationAdapter a).typeVariableModel(name, path)
   or
   Extensions::typeVariableModel(name, path)
 }
 
 /**
- * Holds if CSV rows involving `type` might be relevant for the analysis of this database.
+ * Holds if rows involving `type` might be relevant for the analysis of this database.
  */
 predicate isRelevantType(string type) {
   (
-    sourceModel(type, _, _) or
-    sinkModel(type, _, _) or
-    summaryModel(type, _, _, _, _) or
+    sourceModel(type, _, _, _) or
+    sinkModel(type, _, _, _) or
+    summaryModel(type, _, _, _, _, _) or
     typeModel(_, type, _)
   ) and
   (
@@ -313,26 +378,26 @@ predicate isRelevantType(string type) {
 }
 
 /**
- * Holds if `type,path` is used in some CSV row.
+ * Holds if `type,path` is used in some row.
  */
 pragma[nomagic]
 predicate isRelevantFullPath(string type, string path) {
   isRelevantType(type) and
   (
-    sourceModel(type, path, _) or
-    sinkModel(type, path, _) or
-    summaryModel(type, path, _, _, _) or
+    sourceModel(type, path, _, _) or
+    sinkModel(type, path, _, _) or
+    summaryModel(type, path, _, _, _, _) or
     typeModel(_, type, path)
   )
 }
 
-/** A string from a CSV row that should be parsed as an access path. */
+/** A string from a row that should be parsed as an access path. */
 private predicate accessPathRange(string s) {
   isRelevantFullPath(_, s)
   or
   exists(string type | isRelevantType(type) |
-    summaryModel(type, _, s, _, _) or
-    summaryModel(type, _, _, s, _)
+    summaryModel(type, _, s, _, _, _) or
+    summaryModel(type, _, _, s, _, _)
   )
   or
   typeVariableModel(_, s)
@@ -435,7 +500,7 @@ private API::Node getNodeFromType(string type) {
  * Gets the API node identified by the first `n` tokens of `path` in the given `(type, path)` tuple.
  */
 pragma[nomagic]
-private API::Node getNodeFromPath(string type, AccessPath path, int n) {
+API::Node getNodeFromPath(string type, AccessPath path, int n) {
   isRelevantFullPath(type, path) and
   (
     n = 0 and
@@ -543,7 +608,7 @@ private API::Node getNodeFromPath(string type, AccessPath path) {
 
 pragma[nomagic]
 private predicate typeStepModel(string type, AccessPath basePath, AccessPath output) {
-  summaryModel(type, basePath, "", output, "type")
+  summaryModel(type, basePath, "", output, "type", _)
 }
 
 pragma[nomagic]
@@ -618,36 +683,36 @@ module ModelOutput {
   cached
   private module Cached {
     /**
-     * Holds if a CSV source model contributed `source` with the given `kind`.
+     * Holds if a source model contributed `source` with the given `kind`.
      */
     cached
-    API::Node getASourceNode(string kind) {
+    API::Node getASourceNode(string kind, string model) {
       exists(string type, string path |
-        sourceModel(type, path, kind) and
+        sourceModel(type, path, kind, model) and
         result = getNodeFromPath(type, path)
       )
     }
 
     /**
-     * Holds if a CSV sink model contributed `sink` with the given `kind`.
+     * Holds if a sink model contributed `sink` with the given `kind`.
      */
     cached
-    API::Node getASinkNode(string kind) {
+    API::Node getASinkNode(string kind, string model) {
       exists(string type, string path |
-        sinkModel(type, path, kind) and
+        sinkModel(type, path, kind, model) and
         result = getNodeFromPath(type, path)
       )
     }
 
     /**
-     * Holds if a relevant CSV summary exists for these parameters.
+     * Holds if a relevant summary exists for these parameters.
      */
     cached
     predicate relevantSummaryModel(
-      string type, string path, string input, string output, string kind
+      string type, string path, string input, string output, string kind, string model
     ) {
       isRelevantType(type) and
-      summaryModel(type, path, input, output, kind)
+      summaryModel(type, path, input, output, kind, model)
     }
 
     /**
@@ -655,7 +720,7 @@ module ModelOutput {
      */
     cached
     predicate resolvedSummaryBase(string type, string path, Specific::InvokeNode baseNode) {
-      summaryModel(type, path, _, _, _) and
+      summaryModel(type, path, _, _, _, _) and
       baseNode = getInvocationFromPath(type, path)
     }
 
@@ -664,13 +729,13 @@ module ModelOutput {
      */
     cached
     predicate resolvedSummaryRefBase(string type, string path, API::Node baseNode) {
-      summaryModel(type, path, _, _, _) and
+      summaryModel(type, path, _, _, _, _) and
       baseNode = getNodeFromPath(type, path)
     }
 
     /**
      * Holds if `node` is seen as an instance of `type` due to a type definition
-     * contributed by a CSV model.
+     * contributed by a model.
      */
     cached
     API::Node getATypeNode(string type) { result = getNodeFromType(type) }
@@ -680,12 +745,22 @@ module ModelOutput {
   import Specific::ModelOutputSpecific
   private import codeql.mad.ModelValidation as SharedModelVal
 
+  /**
+   * Holds if a CSV source model contributed `source` with the given `kind`.
+   */
+  API::Node getASourceNode(string kind) { result = getASourceNode(kind, _) }
+
+  /**
+   * Holds if a CSV sink model contributed `sink` with the given `kind`.
+   */
+  API::Node getASinkNode(string kind) { result = getASinkNode(kind, _) }
+
   private module KindValConfig implements SharedModelVal::KindValidationConfigSig {
-    predicate summaryKind(string kind) { summaryModel(_, _, _, _, kind) }
+    predicate summaryKind(string kind) { summaryModel(_, _, _, _, kind, _) }
 
-    predicate sinkKind(string kind) { sinkModel(_, _, kind) }
+    predicate sinkKind(string kind) { sinkModel(_, _, kind, _) }
 
-    predicate sourceKind(string kind) { sourceModel(_, _, kind) }
+    predicate sourceKind(string kind) { sourceModel(_, _, kind, _) }
   }
 
   private module KindVal = SharedModelVal::KindValidation<KindValConfig>;
@@ -694,25 +769,6 @@ module ModelOutput {
    * Gets an error message relating to an invalid CSV row in a model.
    */
   string getAWarning() {
-    // Check number of columns
-    exists(string row, string kind, int expectedArity, int actualArity |
-      any(SourceModelCsv csv).row(row) and kind = "source" and expectedArity = 3
-      or
-      any(SinkModelCsv csv).row(row) and kind = "sink" and expectedArity = 3
-      or
-      any(SummaryModelCsv csv).row(row) and kind = "summary" and expectedArity = 5
-      or
-      any(TypeModelCsv csv).row(row) and kind = "type" and expectedArity = 3
-      or
-      any(TypeVariableModelCsv csv).row(row) and kind = "type-variable" and expectedArity = 2
-    |
-      actualArity = count(row.indexOf(";")) + 1 and
-      actualArity != expectedArity and
-      result =
-        "CSV " + kind + " row should have " + expectedArity + " columns but has " + actualArity +
-          ": " + row
-    )
-    or
     // Check names and arguments of access path tokens
     exists(AccessPath path, AccessPathToken token |
       (isRelevantFullPath(_, path) or typeVariableModel(_, path)) and

python/ql/lib/semmle/python/frameworks/data/internal/ApiGraphModelsExtensions.qll

@@ -8,13 +8,15 @@
  *
  * The kind `remote` represents a general remote flow source.
  */
-extensible predicate sourceModel(string type, string path, string kind);
+extensible predicate sourceModel(
+  string type, string path, string kind, QlBuiltins::ExtensionId madId
+);
 
 /**
  * Holds if the value at `(type, path)` should be seen as a sink
  * of the given `kind`.
  */
-extensible predicate sinkModel(string type, string path, string kind);
+extensible predicate sinkModel(string type, string path, string kind, QlBuiltins::ExtensionId madId);
 
 /**
  * Holds if in calls to `(type, path)`, the value referred to by `input`
@@ -23,7 +25,9 @@ extensible predicate sinkModel(string type, string path, string kind);
  * `kind` should be either `value` or `taint`, for value-preserving or taint-preserving steps,
  * respectively.
  */
-extensible predicate summaryModel(string type, string path, string input, string output, string kind);
+extensible predicate summaryModel(
+  string type, string path, string input, string output, string kind, QlBuiltins::ExtensionId madId
+);
 
 /**
  * Holds if calls to `(type, path)` should be considered neutral. The meaning of this depends on the `kind`.

python/ql/lib/semmle/python/frameworks/data/internal/ApiGraphModelsSpecific.qll

@@ -138,7 +138,7 @@ predicate invocationMatchesExtraCallSiteFilter(API::CallNode invoke, AccessPathT
 pragma[nomagic]
 private predicate relevantInputOutputPath(API::CallNode base, AccessPath inputOrOutput) {
   exists(string type, string input, string output, string path |
-    ModelOutput::relevantSummaryModel(type, path, input, output, _) and
+    ModelOutput::relevantSummaryModel(type, path, input, output, _, _) and
     ModelOutput::resolvedSummaryBase(type, path, base) and
     inputOrOutput = [input, output]
   )
@@ -170,7 +170,7 @@ private API::Node getNodeFromInputOutputPath(API::CallNode baseNode, AccessPath
  */
 predicate summaryStep(API::Node pred, API::Node succ, string kind) {
   exists(string type, string path, API::CallNode base, AccessPath input, AccessPath output |
-    ModelOutput::relevantSummaryModel(type, path, input, output, kind) and
+    ModelOutput::relevantSummaryModel(type, path, input, output, kind, _) and // TODO???
     ModelOutput::resolvedSummaryBase(type, path, base) and
     pred = getNodeFromInputOutputPath(base, input) and
     succ = getNodeFromInputOutputPath(base, output)

python/ql/lib/semmle/python/frameworks/data/internal/empty.model.yml

@@ -1,5 +1,6 @@
 extensions:
-  # Contribute empty data sets to avoid errors about an undefined extensionals
+  # Make sure that the extensible model predicates have at least one definition
+  # to avoid errors about undefined extensionals.
   - addsTo:
       pack: codeql/python-all
       extensible: sourceModel