Merge branch 'main' into gradio-model

python/ql/lib/semmle/python/ApiGraphs.qll

@@ -257,7 +257,7 @@ module API {
      */
     Node getSubscript(string key) {
       exists(API::Node index | result = this.getSubscriptAt(index) |
-        key = index.getAValueReachingSink().asExpr().(PY::StrConst).getText()
+        key = index.getAValueReachingSink().asExpr().(PY::StringLiteral).getText()
       )
     }
 

python/ql/lib/semmle/python/Concepts.qll

@@ -855,7 +855,7 @@ module Http {
 
         /** Gets the URL pattern for this route, if it can be statically determined. */
         string getUrlPattern() {
-          exists(StrConst str |
+          exists(StringLiteral str |
             this.getUrlPatternArg().getALocalSource() = DataFlow::exprNode(str) and
             result = str.getText()
           )
@@ -983,7 +983,7 @@ module Http {
 
         /** Gets the mimetype of this HTTP response, if it can be statically determined. */
         string getMimetype() {
-          exists(StrConst str |
+          exists(StringLiteral str |
             this.getMimetypeOrContentTypeArg().getALocalSource() = DataFlow::exprNode(str) and
             result = str.getText().splitAt(";", 0)
           )

python/ql/lib/semmle/python/Exprs.qll

@@ -236,7 +236,7 @@ class Call extends Call_ {
   string getANamedArgumentName() {
     result = this.getAKeyword().getArg()
     or
-    result = this.getKwargs().(Dict).getAKey().(StrConst).getText()
+    result = this.getKwargs().(Dict).getAKey().(StringLiteral).getText()
   }
 
   /** Gets the positional argument count of this call, provided there is no more than one tuple (*) argument. */
@@ -299,7 +299,7 @@ class Repr extends Repr_ {
  * A bytes constant, such as `b'ascii'`. Note that unadorned string constants such as
  * `"hello"` are treated as Bytes for Python2, but Unicode for Python3.
  */
-class Bytes extends StrConst {
+class Bytes extends StringLiteral {
   /* syntax: b"hello" */
   Bytes() { not this.isUnicode() }
 
@@ -446,7 +446,7 @@ class NegativeIntegerLiteral extends ImmutableLiteral, UnaryExpr {
  * A unicode string expression, such as `u"\u20ac"`. Note that unadorned string constants such as
  * "hello" are treated as Bytes for Python2, but Unicode for Python3.
  */
-class Unicode extends StrConst {
+class Unicode extends StringLiteral {
   /* syntax: "hello" */
   Unicode() { this.isUnicode() }
 
@@ -599,7 +599,7 @@ class Slice extends Slice_ {
 /**
  * Returns all string prefixes in the database that are explicitly marked as Unicode strings.
  *
- * Helper predicate for `StrConst::isUnicode`.
+ * Helper predicate for `StringLiteral::isUnicode`.
  */
 pragma[nomagic]
 private string unicode_prefix() {
@@ -610,7 +610,7 @@ private string unicode_prefix() {
 /**
  * Returns all string prefixes in the database that are _not_ explicitly marked as bytestrings.
  *
- * Helper predicate for `StrConst::isUnicode`.
+ * Helper predicate for `StringLiteral::isUnicode`.
  */
 pragma[nomagic]
 private string non_byte_prefix() {
@@ -618,12 +618,19 @@ private string non_byte_prefix() {
   not result.charAt(_) in ["b", "B"]
 }
 
-/** A string constant. This is a placeholder class -- use `StrConst` instead. */
-class Str = StrConst;
+/** DEPRECATED. Use `StringLiteral` instead. */
+deprecated class Str = StringLiteral;
+
+/** DEPRECATED. Use `StringLiteral` instead. */
+deprecated class StrConst = StringLiteral;
 
 /** A string constant. */
-class StrConst extends Str_, ImmutableLiteral {
+class StringLiteral extends Str_, ImmutableLiteral {
   /* syntax: "hello" */
+  /**
+   * Holds if this string is a unicode string, either by default (e.g. if Python 3), or with an
+   * explicit prefix.
+   */
   predicate isUnicode() {
     this.getPrefix() = unicode_prefix()
     or
@@ -652,6 +659,8 @@ class StrConst extends Str_, ImmutableLiteral {
   }
 
   override Object getLiteralObject() { none() }
+
+  override string toString() { result = "StringLiteral" }
 }
 
 private predicate name_consts(Name_ n, string id) {

python/ql/lib/semmle/python/Files.qll

@@ -93,7 +93,7 @@ class File extends Container, Impl::File {
       exists(Stmt s | s.getLocation().getFile() = this)
       or
       // The file contains the usual `if __name__ == '__main__':` construction
-      exists(If i, Name name, StrConst main, Cmpop op |
+      exists(If i, Name name, StringLiteral main, Cmpop op |
         i.getScope().(Module).getFile() = this and
         op instanceof Eq and
         i.getTest().(Compare).compares(name, op, main) and
@@ -123,7 +123,7 @@ private predicate occupied_line(File f, int n) {
   exists(Location l | l.getFile() = f |
     l.getStartLine() = n
     or
-    exists(StrConst s | s.getLocation() = l | n in [l.getStartLine() .. l.getEndLine()])
+    exists(StringLiteral s | s.getLocation() = l | n in [l.getStartLine() .. l.getEndLine()])
   )
 }
 

python/ql/lib/semmle/python/Module.qll

@@ -125,9 +125,9 @@ class Module extends Module_, Scope, AstNode {
       a.getScope() = this and
       all.getId() = "__all__" and
       (
-        a.getValue().(List).getAnElt().(StrConst).getText() = name
+        a.getValue().(List).getAnElt().(StringLiteral).getText() = name
         or
-        a.getValue().(Tuple).getAnElt().(StrConst).getText() = name
+        a.getValue().(Tuple).getAnElt().(StringLiteral).getText() = name
       )
     )
   }

python/ql/lib/semmle/python/PrintAst.qll

@@ -423,13 +423,13 @@ class ParameterNode extends AstElementNode {
 }
 
 /**
- * A print node for a `StrConst`.
+ * A print node for a `StringLiteral`.
  *
  * The string has a child, if the child is used as a regular expression,
  * which is the root of the regular expression.
  */
-class StrConstNode extends AstElementNode {
-  override StrConst element;
+class StringLiteralNode extends AstElementNode {
+  override StringLiteral element;
 }
 
 /**
@@ -599,7 +599,7 @@ private module PrettyPrinting {
       or
       result = "class " + a.(Class).getName()
       or
-      result = a.(StrConst).getText()
+      result = a.(StringLiteral).getText()
       or
       result = "yield " + a.(Yield).getValue()
       or

python/ql/lib/semmle/python/Scope.qll

@@ -48,7 +48,7 @@ class Scope extends Scope_ {
   string getName() { py_strs(result, this, 0) }
 
   /** Gets the docstring for this scope */
-  StrConst getDocString() { result = this.getStmt(0).(ExprStmt).getValue() }
+  StringLiteral getDocString() { result = this.getStmt(0).(ExprStmt).getValue() }
 
   /** Gets the entry point into this Scope's control flow graph */
   ControlFlowNode getEntryNode() { py_scope_flow(result, this, -1) }

python/ql/lib/semmle/python/Stmts.qll

@@ -284,7 +284,7 @@ class If extends If_ {
 
   /** Whether this if statement takes the form `if __name__ == "__main__":` */
   predicate isNameEqMain() {
-    exists(StrConst m, Name n, Compare c |
+    exists(StringLiteral m, Name n, Compare c |
       this.getTest() = c and
       c.getOp(0) instanceof Eq and
       (

python/ql/lib/semmle/python/dataflow/new/BarrierGuards.qll

@@ -5,7 +5,7 @@ private import semmle.python.dataflow.new.DataFlow
 
 private predicate stringConstCompare(DataFlow::GuardNode g, ControlFlowNode node, boolean branch) {
   exists(CompareNode cn | cn = g |
-    exists(StrConst str_const, Cmpop op |
+    exists(StringLiteral str_const, Cmpop op |
       op = any(Eq eq) and branch = true
       or
       op = any(NotEq ne) and branch = false
@@ -21,7 +21,7 @@ private predicate stringConstCompare(DataFlow::GuardNode g, ControlFlowNode node
       op = any(NotIn ni) and branch = false
     |
       forall(ControlFlowNode elem | elem = str_const_iterable.getAnElement() |
-        elem.getNode() instanceof StrConst
+        elem.getNode() instanceof StringLiteral
       ) and
       cn.operands(node, op, str_const_iterable)
     )

python/ql/lib/semmle/python/dataflow/new/FlowSummary.qll

@@ -31,8 +31,21 @@ abstract class SummarizedCallable extends LibraryCallable, Impl::Public::Summari
    * DEPRECATED: Use `propagatesFlow` instead.
    */
   deprecated predicate propagatesFlowExt(string input, string output, boolean preservesValue) {
-    this.propagatesFlow(input, output, preservesValue)
+    this.propagatesFlow(input, output, preservesValue, _)
   }
+
+  override predicate propagatesFlow(
+    string input, string output, boolean preservesValue, string model
+  ) {
+    this.propagatesFlow(input, output, preservesValue) and model = this
+  }
+
+  /**
+   * Holds if data may flow from `input` to `output` through this callable.
+   *
+   * `preservesValue` indicates whether this is a value-preserving step or a taint-step.
+   */
+  predicate propagatesFlow(string input, string output, boolean preservesValue) { none() }
 }
 
 deprecated class RequiredSummaryComponentStack = Impl::Private::RequiredSummaryComponentStack;

python/ql/lib/semmle/python/dataflow/new/Regexp.qll

@@ -7,20 +7,6 @@ private import semmle.python.regex
 private import semmle.python.dataflow.new.DataFlow
 private import semmle.python.regexp.internal.RegExpTracking
 
-/**
- * Provides utility predicates related to regular expressions.
- */
-deprecated module RegExpPatterns {
-  /**
-   * Gets a pattern that matches common top-level domain names in lower case.
-   * DEPRECATED: use the similarly named predicate from `HostnameRegex` from the `regex` pack instead.
-   */
-  deprecated string getACommonTld() {
-    // according to ranking by http://google.com/search?q=site:.<<TLD>>
-    result = "(?:com|org|edu|gov|uk|net|io)(?![a-z0-9])"
-  }
-}
-
 /**
  * A node whose value may flow to a position where it is interpreted
  * as a part of a regular expression.

python/ql/lib/semmle/python/dataflow/new/SensitiveDataSources.qll

@@ -89,9 +89,9 @@ private module SensitiveDataModeling {
    */
   DataFlow::Node sensitiveLookupStringConst(SensitiveDataClassification classification) {
     // Note: If this is implemented with type-tracking, we will get cross-talk as
-    // illustrated in python/ql/test/experimental/dataflow/sensitive-data/test.py
+    // illustrated in python/ql/test/library-tests/dataflow/sensitive-data/test.py
     exists(DataFlow::LocalSourceNode source |
-      source.asExpr().(StrConst).getText() = sensitiveString(classification) and
+      source.asExpr().(StringLiteral).getText() = sensitiveString(classification) and
       source.flowsTo(result)
     )
   }
@@ -173,8 +173,8 @@ private module SensitiveDataModeling {
   }
 
   pragma[nomagic]
-  private string sensitiveStrConstCandidate() {
-    result = any(StrConst s | not s.isDocString()).getText() and
+  private string sensitiveStringLiteralCandidate() {
+    result = any(StringLiteral s | not s.isDocString()).getText() and
     not result.regexpMatch(notSensitiveRegexp())
   }
 
@@ -217,7 +217,7 @@ private module SensitiveDataModeling {
     result in [
         sensitiveNameCandidate(), sensitiveAttributeNameCandidate(),
         sensitiveParameterNameCandidate(), sensitiveFunctionNameCandidate(),
-        sensitiveStrConstCandidate()
+        sensitiveStringLiteralCandidate()
       ]
   }
 

python/ql/lib/semmle/python/dataflow/new/TypeTracking.qll

@@ -5,9 +5,14 @@
 
 private import internal.TypeTrackingImpl as Impl
 import Impl::Shared::TypeTracking<Impl::TypeTrackingInput>
+private import semmle.python.dataflow.new.internal.DataFlowPublic as DataFlowPublic
 
-/** A string that may appear as the name of an attribute or access path. */
-class AttributeName = Impl::TypeTrackingInput::Content;
+/**
+ * DEPRECATED.
+ *
+ * A string that may appear as the name of an attribute or access path.
+ */
+deprecated class AttributeName = Impl::TypeTrackingInput::Content;
 
 /**
  * A summary of the steps needed to track a value to a given dataflow node.
@@ -40,7 +45,11 @@ class TypeTracker extends Impl::TypeTracker {
    * Holds if this is the starting point of type tracking, and the value starts in the attribute named `attrName`.
    * The type tracking only ends after the attribute has been loaded.
    */
-  predicate startInAttr(string attrName) { this.startInContent(attrName) }
+  predicate startInAttr(string attrName) {
+    exists(DataFlowPublic::AttributeContent content | content.getAttribute() = attrName |
+      this.startInContent(content)
+    )
+  }
 
   /**
    * INTERNAL. DO NOT USE.
@@ -48,9 +57,8 @@ class TypeTracker extends Impl::TypeTracker {
    * Gets the attribute associated with this type tracker.
    */
   string getAttr() {
-    result = this.getContent().asSome()
-    or
-    this.getContent().isNone() and
-    result = ""
+    if this.getContent().asSome() instanceof DataFlowPublic::AttributeContent
+    then result = this.getContent().asSome().(DataFlowPublic::AttributeContent).getAttribute()
+    else result = ""
   }
 }

python/ql/lib/semmle/python/dataflow/new/internal/Attributes.qll

@@ -40,7 +40,7 @@ abstract class AttrRef extends Node {
     or
     exists(LocalSourceNode nodeFrom |
       nodeFrom.flowsTo(this.getAttributeNameExpr()) and
-      attrName = nodeFrom.(CfgNode).getNode().getNode().(StrConst).getText()
+      attrName = nodeFrom.(CfgNode).getNode().getNode().(StringLiteral).getText()
     )
   }
 
@@ -178,7 +178,7 @@ private class SetAttrCallAsAttrWrite extends AttrWrite, CfgNode {
   override ExprNode getAttributeNameExpr() { result.asCfgNode() = node.getName() }
 
   override string getAttributeName() {
-    result = this.getAttributeNameExpr().(CfgNode).getNode().getNode().(StrConst).getText()
+    result = this.getAttributeNameExpr().(CfgNode).getNode().getNode().(StringLiteral).getText()
   }
 }
 
@@ -254,7 +254,7 @@ private class GetAttrCallAsAttrRead extends AttrRead, CfgNode {
   override ExprNode getAttributeNameExpr() { result.asCfgNode() = node.getName() }
 
   override string getAttributeName() {
-    result = this.getAttributeNameExpr().(CfgNode).getNode().getNode().(StrConst).getText()
+    result = this.getAttributeNameExpr().(CfgNode).getNode().getNode().(StringLiteral).getText()
   }
 }
 

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl1.qll

@@ -263,9 +263,10 @@ deprecated private module Config implements FullStateConfigSig {
 
   predicate isBarrierOut(Node node, FlowState state) { none() }
 
-  predicate isAdditionalFlowStep(Node node1, Node node2) {
+  predicate isAdditionalFlowStep(Node node1, Node node2, string model) {
     singleConfiguration() and
-    any(Configuration config).isAdditionalFlowStep(node1, node2)
+    any(Configuration config).isAdditionalFlowStep(node1, node2) and
+    model = ""
   }
 
   predicate isAdditionalFlowStep(Node node1, FlowState state1, Node node2, FlowState state2) {

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl2.qll

@@ -263,9 +263,10 @@ deprecated private module Config implements FullStateConfigSig {
 
   predicate isBarrierOut(Node node, FlowState state) { none() }
 
-  predicate isAdditionalFlowStep(Node node1, Node node2) {
+  predicate isAdditionalFlowStep(Node node1, Node node2, string model) {
     singleConfiguration() and
-    any(Configuration config).isAdditionalFlowStep(node1, node2)
+    any(Configuration config).isAdditionalFlowStep(node1, node2) and
+    model = ""
   }
 
   predicate isAdditionalFlowStep(Node node1, FlowState state1, Node node2, FlowState state2) {

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl3.qll

@@ -263,9 +263,10 @@ deprecated private module Config implements FullStateConfigSig {
 
   predicate isBarrierOut(Node node, FlowState state) { none() }
 
-  predicate isAdditionalFlowStep(Node node1, Node node2) {
+  predicate isAdditionalFlowStep(Node node1, Node node2, string model) {
     singleConfiguration() and
-    any(Configuration config).isAdditionalFlowStep(node1, node2)
+    any(Configuration config).isAdditionalFlowStep(node1, node2) and
+    model = ""
   }
 
   predicate isAdditionalFlowStep(Node node1, FlowState state1, Node node2, FlowState state2) {

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl4.qll

@@ -263,9 +263,10 @@ deprecated private module Config implements FullStateConfigSig {
 
   predicate isBarrierOut(Node node, FlowState state) { none() }
 
-  predicate isAdditionalFlowStep(Node node1, Node node2) {
+  predicate isAdditionalFlowStep(Node node1, Node node2, string model) {
     singleConfiguration() and
-    any(Configuration config).isAdditionalFlowStep(node1, node2)
+    any(Configuration config).isAdditionalFlowStep(node1, node2) and
+    model = ""
   }
 
   predicate isAdditionalFlowStep(Node node1, FlowState state1, Node node2, FlowState state2) {

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowPrivate.qll

@@ -3,6 +3,7 @@ private import DataFlowPublic
 private import semmle.python.essa.SsaCompute
 private import semmle.python.dataflow.new.internal.ImportResolution
 private import FlowSummaryImpl as FlowSummaryImpl
+private import semmle.python.frameworks.data.ModelsAsData
 // Since we allow extra data-flow steps from modeled frameworks, we import these
 // up-front, to ensure these are included. This provides a more seamless experience from
 // a user point of view, since they don't need to know they need to import a specific
@@ -471,12 +472,12 @@ import StepRelationTransformations
  *
  * It includes flow steps from flow summaries.
  */
-predicate simpleLocalFlowStep(Node nodeFrom, Node nodeTo) {
-  simpleLocalFlowStepForTypetracking(nodeFrom, nodeTo)
+predicate simpleLocalFlowStep(Node nodeFrom, Node nodeTo, string model) {
+  simpleLocalFlowStepForTypetracking(nodeFrom, nodeTo) and model = ""
   or
-  summaryLocalStep(nodeFrom, nodeTo)
+  summaryLocalStep(nodeFrom, nodeTo, model)
   or
-  variableCaptureLocalFlowStep(nodeFrom, nodeTo)
+  variableCaptureLocalFlowStep(nodeFrom, nodeTo) and model = ""
 }
 
 /**
@@ -490,9 +491,9 @@ predicate simpleLocalFlowStepForTypetracking(Node nodeFrom, Node nodeTo) {
   LocalFlow::localFlowStep(nodeFrom, nodeTo)
 }
 
-private predicate summaryLocalStep(Node nodeFrom, Node nodeTo) {
+private predicate summaryLocalStep(Node nodeFrom, Node nodeTo, string model) {
   FlowSummaryImpl::Private::Steps::summaryLocalStep(nodeFrom.(FlowSummaryNode).getSummaryNode(),
-    nodeTo.(FlowSummaryNode).getSummaryNode(), true)
+    nodeTo.(FlowSummaryNode).getSummaryNode(), true, model)
 }
 
 predicate variableCaptureLocalFlowStep(Node nodeFrom, Node nodeTo) {
@@ -642,23 +643,37 @@ predicate jumpStepNotSharedWithTypeTracker(Node nodeFrom, Node nodeTo) {
 // Field flow
 //--------
 /**
- * Holds if data can flow from `nodeFrom` to `nodeTo` via an assignment to
- * content `c`.
+ * Subset of `storeStep` that should be shared with type-tracking.
+ *
+ * NOTE: This does not include attributeStoreStep right now, since it has its' own
+ * modeling in the type-tracking library (which is slightly different due to
+ * PostUpdateNodes).
+ *
+ * As of 2024-04-02 the type-tracking library only supports precise content, so there is
+ * no reason to include steps for list content right now.
  */
-predicate storeStep(Node nodeFrom, ContentSet c, Node nodeTo) {
-  listStoreStep(nodeFrom, c, nodeTo)
-  or
-  setStoreStep(nodeFrom, c, nodeTo)
-  or
+predicate storeStepCommon(Node nodeFrom, ContentSet c, Node nodeTo) {
   tupleStoreStep(nodeFrom, c, nodeTo)
   or
   dictStoreStep(nodeFrom, c, nodeTo)
   or
   moreDictStoreSteps(nodeFrom, c, nodeTo)
   or
-  comprehensionStoreStep(nodeFrom, c, nodeTo)
-  or
   iterableUnpackingStoreStep(nodeFrom, c, nodeTo)
+}
+
+/**
+ * Holds if data can flow from `nodeFrom` to `nodeTo` via an assignment to
+ * content `c`.
+ */
+predicate storeStep(Node nodeFrom, ContentSet c, Node nodeTo) {
+  storeStepCommon(nodeFrom, c, nodeTo)
+  or
+  listStoreStep(nodeFrom, c, nodeTo)
+  or
+  setStoreStep(nodeFrom, c, nodeTo)
+  or
+  comprehensionStoreStep(nodeFrom, c, nodeTo)
   or
   attributeStoreStep(nodeFrom, c, nodeTo)
   or
@@ -798,7 +813,7 @@ predicate dictStoreStep(CfgNode nodeFrom, DictionaryElementContent c, Node nodeT
   exists(KeyValuePair item |
     item = nodeTo.asCfgNode().(DictNode).getNode().(Dict).getAnItem() and
     nodeFrom.getNode().getNode() = item.getValue() and
-    c.getKey() = item.getKey().(StrConst).getS()
+    c.getKey() = item.getKey().(StringLiteral).getS()
   )
 }
 
@@ -814,13 +829,13 @@ private predicate moreDictStoreSteps(CfgNode nodeFrom, DictionaryElementContent
   exists(SubscriptNode subscript |
     nodeTo.(PostUpdateNode).getPreUpdateNode().asCfgNode() = subscript.getObject() and
     nodeFrom.asCfgNode() = subscript.(DefinitionNode).getValue() and
-    c.getKey() = subscript.getIndex().getNode().(StrConst).getText()
+    c.getKey() = subscript.getIndex().getNode().(StringLiteral).getText()
   )
   or
   // see https://docs.python.org/3.10/library/stdtypes.html#dict.setdefault
   exists(MethodCallNode call |
     call.calls(nodeTo.(PostUpdateNode).getPreUpdateNode(), "setdefault") and
-    call.getArg(0).asExpr().(StrConst).getText() = c.getKey() and
+    call.getArg(0).asExpr().(StringLiteral).getText() = c.getKey() and
     nodeFrom = call.getArg(1)
   )
 }
@@ -829,7 +844,7 @@ predicate dictClearStep(Node node, DictionaryElementContent c) {
   exists(SubscriptNode subscript |
     subscript instanceof DefinitionNode and
     node.asCfgNode() = subscript.getObject() and
-    c.getKey() = subscript.getIndex().getNode().(StrConst).getText()
+    c.getKey() = subscript.getIndex().getNode().(StringLiteral).getText()
   )
 }
 
@@ -892,12 +907,19 @@ predicate attributeStoreStep(Node nodeFrom, AttributeContent c, Node nodeTo) {
 }
 
 /**
- * Holds if data can flow from `nodeFrom` to `nodeTo` via a read of content `c`.
+ * Subset of `readStep` that should be shared with type-tracking.
  */
-predicate readStep(Node nodeFrom, ContentSet c, Node nodeTo) {
+predicate readStepCommon(Node nodeFrom, ContentSet c, Node nodeTo) {
   subscriptReadStep(nodeFrom, c, nodeTo)
   or
   iterableUnpackingReadStep(nodeFrom, c, nodeTo)
+}
+
+/**
+ * Holds if data can flow from `nodeFrom` to `nodeTo` via a read of content `c`.
+ */
+predicate readStep(Node nodeFrom, ContentSet c, Node nodeTo) {
+  readStepCommon(nodeFrom, c, nodeTo)
   or
   matchReadStep(nodeFrom, c, nodeTo)
   or
@@ -932,7 +954,7 @@ predicate subscriptReadStep(CfgNode nodeFrom, Content c, CfgNode nodeTo) {
       nodeTo.getNode().(SubscriptNode).getIndex().getNode().(IntegerLiteral).getValue()
     or
     c.(DictionaryElementContent).getKey() =
-      nodeTo.getNode().(SubscriptNode).getIndex().getNode().(StrConst).getS()
+      nodeTo.getNode().(SubscriptNode).getIndex().getNode().(StringLiteral).getS()
   )
 }
 
@@ -1057,6 +1079,16 @@ predicate lambdaCall(DataFlowCall call, LambdaCallKind kind, Node receiver) {
 /** Extra data-flow steps needed for lambda flow analysis. */
 predicate additionalLambdaFlowStep(Node nodeFrom, Node nodeTo, boolean preservesValue) { none() }
 
+predicate knownSourceModel(Node source, string model) {
+  source = ModelOutput::getASourceNode(_, model).asSource()
+}
+
+predicate knownSinkModel(Node sink, string model) {
+  sink = ModelOutput::getASinkNode(_, model).asSink()
+}
+
+class DataFlowSecondLevelScope = Unit;
+
 /**
  * Holds if flow is allowed to pass from parameter `p` and back to itself as a
  * side-effect, resulting in a summary from `p` to itself.

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowPublic.qll

@@ -606,17 +606,18 @@ newtype TContent =
   /** An element of a dictionary under a specific key. */
   TDictionaryElementContent(string key) {
     // {"key": ...}
-    key = any(KeyValuePair kvp).getKey().(StrConst).getText()
+    key = any(KeyValuePair kvp).getKey().(StringLiteral).getText()
     or
     // func(key=...)
     key = any(Keyword kw).getArg()
     or
     // d["key"] = ...
-    key = any(SubscriptNode sub | sub.isStore() | sub.getIndex().getNode().(StrConst).getText())
+    key =
+      any(SubscriptNode sub | sub.isStore() | sub.getIndex().getNode().(StringLiteral).getText())
     or
     // d.setdefault("key", ...)
     exists(CallNode call | call.getFunction().(AttrNode).getName() = "setdefault" |
-      key = call.getArg(0).getNode().(StrConst).getText()
+      key = call.getArg(0).getNode().(StringLiteral).getText()
     )
   } or
   /** An element of a dictionary under any key. */
@@ -637,12 +638,14 @@ newtype TContent =
     //   name = any(AccessPathToken a).getAnArgument("Attribute")
     // instead we use a qltest to alert if we write a new summary in QL that uses an
     // attribute -- see
-    // python/ql/test/experimental/dataflow/summaries-checks/missing-attribute-content.ql
+    // python/ql/test/library-tests/dataflow/summaries-checks/missing-attribute-content.ql
     attr in ["re", "string", "pattern"]
     or
     //
     // 2) summaries in data-extension files
-    exists(string input, string output | ModelOutput::relevantSummaryModel(_, _, input, output, _) |
+    exists(string input, string output |
+      ModelOutput::relevantSummaryModel(_, _, input, output, _, _)
+    |
       attr = [input, output].regexpFind("(?<=(^|\\.)Attribute\\[)[^\\]]+(?=\\])", _, _).trim()
     )
   } or

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowUtil.qll

@@ -12,7 +12,7 @@ private import FlowSummaryImpl as FlowSummaryImpl
  * (intra-procedural) step.
  */
 predicate localFlowStep(Node nodeFrom, Node nodeTo) {
-  simpleLocalFlowStep(nodeFrom, nodeTo)
+  simpleLocalFlowStep(nodeFrom, nodeTo, _)
   or
   // Simple flow through library code is included in the exposed local
   // step relation, even though flow is technically inter-procedural.

python/ql/lib/semmle/python/dataflow/new/internal/ImportResolution.qll

@@ -146,7 +146,7 @@ module ImportResolution {
       def.getValue() = n and
       def.(NameNode).getId() = "__all__" and
       def.getScope() = m and
-      any(StrConst s | s.getText() = name) = n.getAnElement().getNode()
+      any(StringLiteral s | s.getText() = name) = n.getAnElement().getNode()
     )
   }
 
@@ -210,7 +210,7 @@ module ImportResolution {
       exists(SubscriptNode sub |
         sub.getObject() = sys_modules_reference().asCfgNode() and
         sub.getIndex() = n and
-        n.getNode().(StrConst).getText() = name and
+        n.getNode().(StringLiteral).getText() = name and
         sub.(DefinitionNode).getValue() = mod.asCfgNode() and
         mod = getModuleReference(result)
       )

python/ql/lib/semmle/python/dataflow/new/internal/LocalSources.qll

@@ -242,7 +242,7 @@ private module Cached {
    */
   pragma[nomagic]
   private predicate localSourceFlowStep(Node nodeFrom, Node nodeTo) {
-    simpleLocalFlowStep(nodeFrom, nodeTo) and
+    simpleLocalFlowStep(nodeFrom, nodeTo, _) and
     not nodeTo = any(ModuleVariableNode v).getARead()
   }
 

python/ql/lib/semmle/python/dataflow/new/internal/MatchUnpacking.qll

@@ -224,7 +224,7 @@ predicate matchMappingReadStep(Node nodeFrom, Content c, Node nodeTo) {
   |
     nodeFrom.(CfgNode).getNode().getNode() = subject and
     nodeTo.(CfgNode).getNode().getNode() = value and
-    c.(DictionaryElementContent).getKey() = key.getLiteral().(StrConst).getText()
+    c.(DictionaryElementContent).getKey() = key.getLiteral().(StringLiteral).getText()
   )
 }
 
@@ -256,7 +256,7 @@ predicate matchMappingClearStep(Node n, Content c) {
     dstar = subject.getAMapping()
   |
     n.(CfgNode).getNode().getNode() = dstar.getTarget() and
-    c.(DictionaryElementContent).getKey() = key.getLiteral().(StrConst).getText()
+    c.(DictionaryElementContent).getKey() = key.getLiteral().(StringLiteral).getText()
   )
 }
 

python/ql/lib/semmle/python/dataflow/new/internal/PrintNode.qll

@@ -18,7 +18,7 @@ private import semmle.python.dataflow.new.internal.DataFlowPrivate as DataFlowPr
  */
 string prettyExpr(Expr e) {
   not e instanceof Num and
-  not e instanceof StrConst and
+  not e instanceof StringLiteral and
   not e instanceof Subscript and
   not e instanceof Call and
   not e instanceof Attribute and
@@ -27,8 +27,8 @@ string prettyExpr(Expr e) {
   result = e.(Num).getN()
   or
   result =
-    e.(StrConst).getPrefix() + e.(StrConst).getText() +
-      e.(StrConst).getPrefix().regexpReplaceAll("[a-zA-Z]+", "")
+    e.(StringLiteral).getPrefix() + e.(StringLiteral).getText() +
+      e.(StringLiteral).getPrefix().regexpReplaceAll("[a-zA-Z]+", "")
   or
   result = prettyExpr(e.(Subscript).getObject()) + "[" + prettyExpr(e.(Subscript).getIndex()) + "]"
   or

python/ql/lib/semmle/python/dataflow/new/internal/TaintTrackingPrivate.qll

@@ -24,10 +24,11 @@ private module Cached {
    * global taint flow configurations.
    */
   cached
-  predicate defaultAdditionalTaintStep(DataFlow::Node nodeFrom, DataFlow::Node nodeTo) {
-    localAdditionalTaintStep(nodeFrom, nodeTo)
+  predicate defaultAdditionalTaintStep(DataFlow::Node nodeFrom, DataFlow::Node nodeTo, string model) {
+    localAdditionalTaintStep(nodeFrom, nodeTo, model)
     or
-    any(AdditionalTaintStep a).step(nodeFrom, nodeTo)
+    any(AdditionalTaintStep a).step(nodeFrom, nodeTo) and
+    model = "AdditionalTaintStep"
   }
 
   /**
@@ -36,30 +37,34 @@ private module Cached {
    * different objects.
    */
   cached
-  predicate localAdditionalTaintStep(DataFlow::Node nodeFrom, DataFlow::Node nodeTo) {
-    concatStep(nodeFrom, nodeTo)
-    or
-    subscriptStep(nodeFrom, nodeTo)
-    or
-    stringManipulation(nodeFrom, nodeTo)
-    or
-    containerStep(nodeFrom, nodeTo)
-    or
-    copyStep(nodeFrom, nodeTo)
-    or
-    DataFlowPrivate::forReadStep(nodeFrom, _, nodeTo)
-    or
-    DataFlowPrivate::iterableUnpackingReadStep(nodeFrom, _, nodeTo)
-    or
-    DataFlowPrivate::iterableUnpackingStoreStep(nodeFrom, _, nodeTo)
-    or
-    awaitStep(nodeFrom, nodeTo)
-    or
-    asyncWithStep(nodeFrom, nodeTo)
+  predicate localAdditionalTaintStep(DataFlow::Node nodeFrom, DataFlow::Node nodeTo, string model) {
+    (
+      concatStep(nodeFrom, nodeTo)
+      or
+      subscriptStep(nodeFrom, nodeTo)
+      or
+      stringManipulation(nodeFrom, nodeTo)
+      or
+      containerStep(nodeFrom, nodeTo)
+      or
+      copyStep(nodeFrom, nodeTo)
+      or
+      DataFlowPrivate::forReadStep(nodeFrom, _, nodeTo)
+      or
+      DataFlowPrivate::iterableUnpackingReadStep(nodeFrom, _, nodeTo)
+      or
+      DataFlowPrivate::iterableUnpackingStoreStep(nodeFrom, _, nodeTo)
+      or
+      awaitStep(nodeFrom, nodeTo)
+      or
+      asyncWithStep(nodeFrom, nodeTo)
+    ) and
+    model = ""
     or
     FlowSummaryImpl::Private::Steps::summaryLocalStep(nodeFrom
           .(DataFlowPrivate::FlowSummaryNode)
-          .getSummaryNode(), nodeTo.(DataFlowPrivate::FlowSummaryNode).getSummaryNode(), false)
+          .getSummaryNode(), nodeTo.(DataFlowPrivate::FlowSummaryNode).getSummaryNode(), false,
+      model)
   }
 }
 

python/ql/lib/semmle/python/dataflow/new/internal/TaintTrackingPublic.qll

@@ -34,7 +34,7 @@ predicate localTaintStep(DataFlow::Node nodeFrom, DataFlow::Node nodeTo) {
   // Ordinary data flow
   DataFlow::localFlowStep(nodeFrom, nodeTo)
   or
-  localAdditionalTaintStep(nodeFrom, nodeTo)
+  localAdditionalTaintStep(nodeFrom, nodeTo, _)
 }
 
 /**

python/ql/lib/semmle/python/dataflow/new/internal/TypeTracker.qll

@@ -1,6 +1,7 @@
 /** Step Summaries and Type Tracking */
 
 private import TypeTrackerSpecific
+private import semmle.python.dataflow.new.internal.DataFlowPublic as DataFlowPublic
 
 cached
 private module Cached {
@@ -12,10 +13,22 @@ private module Cached {
     LevelStep() or
     CallStep() or
     ReturnStep() or
-    deprecated StoreStep(TypeTrackerContent content) { basicStoreStep(_, _, content) } or
-    deprecated LoadStep(TypeTrackerContent content) { basicLoadStep(_, _, content) } or
+    deprecated StoreStep(TypeTrackerContent content) {
+      exists(DataFlowPublic::AttributeContent dfc | dfc.getAttribute() = content |
+        basicStoreStep(_, _, dfc)
+      )
+    } or
+    deprecated LoadStep(TypeTrackerContent content) {
+      exists(DataFlowPublic::AttributeContent dfc | dfc.getAttribute() = content |
+        basicLoadStep(_, _, dfc)
+      )
+    } or
     deprecated LoadStoreStep(TypeTrackerContent load, TypeTrackerContent store) {
-      basicLoadStoreStep(_, _, load, store)
+      exists(DataFlowPublic::AttributeContent dfcLoad, DataFlowPublic::AttributeContent dfcStore |
+        dfcLoad.getAttribute() = load and dfcStore.getAttribute() = store
+      |
+        basicLoadStoreStep(_, _, dfcLoad, dfcStore)
+      )
     } or
     deprecated WithContent(ContentFilter filter) { basicWithContentStep(_, _, filter) } or
     deprecated WithoutContent(ContentFilter filter) { basicWithoutContentStep(_, _, filter) } or
@@ -29,13 +42,13 @@ private module Cached {
       // Restrict `content` to those that might eventually match a load.
       // We can't rely on `basicStoreStep` since `startInContent` might be used with
       // a content that has no corresponding store.
-      exists(TypeTrackerContent loadContents |
+      exists(DataFlowPublic::AttributeContent loadContents |
         (
           basicLoadStep(_, _, loadContents)
           or
           basicLoadStoreStep(_, _, loadContents, _)
         ) and
-        compatibleContents(content, loadContents)
+        compatibleContents(content, loadContents.getAttribute())
       )
     }
 
@@ -45,13 +58,13 @@ private module Cached {
       content = noContent()
       or
       // As in MkTypeTracker, restrict `content` to those that might eventually match a store.
-      exists(TypeTrackerContent storeContent |
+      exists(DataFlowPublic::AttributeContent storeContent |
         (
           basicStoreStep(_, _, storeContent)
           or
           basicLoadStoreStep(_, _, _, storeContent)
         ) and
-        compatibleContents(storeContent, content)
+        compatibleContents(storeContent.getAttribute(), content)
       )
     }
 
@@ -198,7 +211,10 @@ private module Cached {
       flowsToStoreStep(nodeFrom, nodeTo, content) and
       summary = StoreStep(content)
       or
-      basicLoadStep(nodeFrom, nodeTo, content) and summary = LoadStep(content)
+      exists(DataFlowPublic::AttributeContent dfc | dfc.getAttribute() = content |
+        basicLoadStep(nodeFrom, nodeTo, dfc)
+      ) and
+      summary = LoadStep(content)
     )
     or
     exists(TypeTrackerContent loadContent, TypeTrackerContent storeContent |
@@ -281,7 +297,12 @@ deprecated private predicate smallstepProj(Node nodeFrom, StepSummary summary) {
 deprecated private predicate flowsToStoreStep(
   Node nodeFrom, TypeTrackingNode nodeTo, TypeTrackerContent content
 ) {
-  exists(Node obj | nodeTo.flowsTo(obj) and basicStoreStep(nodeFrom, obj, content))
+  exists(Node obj |
+    nodeTo.flowsTo(obj) and
+    exists(DataFlowPublic::AttributeContent dfc | dfc.getAttribute() = content |
+      basicStoreStep(nodeFrom, obj, dfc)
+    )
+  )
 }
 
 /**
@@ -292,7 +313,12 @@ deprecated private predicate flowsToLoadStoreStep(
   TypeTrackerContent storeContent
 ) {
   exists(Node obj |
-    nodeTo.flowsTo(obj) and basicLoadStoreStep(nodeFrom, obj, loadContent, storeContent)
+    nodeTo.flowsTo(obj) and
+    exists(DataFlowPublic::AttributeContent loadDfc, DataFlowPublic::AttributeContent storeDfc |
+      loadDfc.getAttribute() = loadContent and storeDfc.getAttribute() = storeContent
+    |
+      basicLoadStoreStep(nodeFrom, obj, loadDfc, storeDfc)
+    )
   )
 }
 

python/ql/lib/semmle/python/dataflow/new/internal/TypeTrackerSpecific.qll

@@ -15,7 +15,7 @@ deprecated class OptionalTypeTrackerContent extends string {
   OptionalTypeTrackerContent() {
     this = ""
     or
-    this instanceof TypeTrackingImpl::TypeTrackingInput::Content
+    this = any(DataFlowPublic::AttributeContent dfc).getAttribute()
   }
 }
 

python/ql/lib/semmle/python/dataflow/new/internal/TypeTrackingImpl.qll

@@ -8,6 +8,7 @@ private import semmle.python.dataflow.new.internal.DataFlowPrivate as DataFlowPr
 private import codeql.typetracking.internal.SummaryTypeTracker as SummaryTypeTracker
 private import semmle.python.dataflow.new.internal.FlowSummaryImpl as FlowSummaryImpl
 private import semmle.python.dataflow.new.internal.DataFlowDispatch as DataFlowDispatch
+private import semmle.python.dataflow.new.internal.IterableUnpacking as IterableUnpacking
 
 private module SummaryTypeTrackerInput implements SummaryTypeTracker::Input {
   // Dataflow nodes
@@ -23,7 +24,15 @@ private module SummaryTypeTrackerInput implements SummaryTypeTracker::Input {
   ContentFilter getFilterFromWithContentStep(Content content) { none() }
 
   // Callables
-  class SummarizedCallable = FlowSummaryImpl::Private::SummarizedCallableImpl;
+  class SummarizedCallable instanceof FlowSummaryImpl::Private::SummarizedCallableImpl {
+    string toString() { result = super.toString() }
+
+    predicate propagatesFlow(
+      SummaryComponentStack input, SummaryComponentStack output, boolean preservesValue
+    ) {
+      super.propagatesFlow(input, output, preservesValue, _)
+    }
+  }
 
   // Summaries and their stacks
   class SummaryComponent = FlowSummaryImpl::Private::SummaryComponent;
@@ -97,24 +106,25 @@ private module SummaryTypeTrackerInput implements SummaryTypeTracker::Input {
 
 private module TypeTrackerSummaryFlow = SummaryTypeTracker::SummaryFlow<SummaryTypeTrackerInput>;
 
-/**
- * Gets the name of a possible piece of content. For Python, this is currently only attribute names,
- * using the name of the attribute for the corresponding content.
- */
-private string getPossibleContentName() {
-  Stages::TypeTracking::ref() and // the TypeTracking::append() etc. predicates that we want to cache depend on this predicate, so we can place the `ref()` call here to get around identical files.
-  result = any(DataFlowPublic::AttrRef a).getAttributeName()
-}
-
 module TypeTrackingInput implements Shared::TypeTrackingInput {
   class Node = DataFlowPublic::Node;
 
   class LocalSourceNode = DataFlowPublic::LocalSourceNode;
 
-  class Content instanceof string {
-    Content() { this = getPossibleContentName() }
-
-    string toString() { result = this }
+  class Content extends DataFlowPublic::Content {
+    Content() {
+      // TODO: for now, it's not 100% clear if should support non-precise content in
+      // type-tracking, or if it will lead to bad results. We start with only allowing
+      // precise content, which should always be a good improvement! It also simplifies
+      // the process of examining new results from non-precise content steps in the
+      // future, since you will _only_ have to look over the results from the new
+      // non-precise steps.
+      this instanceof DataFlowPublic::AttributeContent
+      or
+      this instanceof DataFlowPublic::DictionaryElementContent
+      or
+      this instanceof DataFlowPublic::TupleElementContent
+    }
   }
 
   /**
@@ -134,7 +144,27 @@ module TypeTrackingInput implements Shared::TypeTrackingInput {
   }
 
   /** Holds if there is a simple local flow step from `nodeFrom` to `nodeTo` */
-  predicate simpleLocalSmallStep = DataFlowPrivate::simpleLocalFlowStepForTypetracking/2;
+  predicate simpleLocalSmallStep(Node nodeFrom, Node nodeTo) {
+    DataFlowPrivate::simpleLocalFlowStepForTypetracking(nodeFrom, nodeTo) and
+    // for `for k,v in foo` no need to do local flow step from the synthetic sequence
+    // node for `k,v` to the tuple `k,v` -- since type-tracking only supports one level
+    // of content tracking, and there is one read-step from `foo` the synthetic sequence
+    // node required, we can skip the flow step from the synthetic sequence node to the
+    // tuple itself, since the read-step from the tuple to the tuple elements will not
+    // matter.
+    not (
+      IterableUnpacking::iterableUnpackingForReadStep(_, _, nodeFrom) and
+      IterableUnpacking::iterableUnpackingTupleFlowStep(nodeFrom, nodeTo)
+    ) and
+    // for nested iterable unpacking, such as `[[a]] = foo` or `((a,b),) = bar`, we can
+    // ignore the flow steps from the synthetic sequence node to the real sequence node,
+    // since we only support one level of content in type-trackers, and the nested
+    // structure requires two levels at least to be useful.
+    not exists(SequenceNode outer |
+      outer.getAnElement() = nodeTo.asCfgNode() and
+      IterableUnpacking::iterableUnpackingTupleFlowStep(nodeFrom, nodeTo)
+    )
+  }
 
   /** Holds if there is a level step from `nodeFrom` to `nodeTo`, which may depend on the call graph. */
   predicate levelStepCall(Node nodeFrom, LocalSourceNode nodeTo) { none() }
@@ -181,46 +211,68 @@ module TypeTrackingInput implements Shared::TypeTrackingInput {
    * Holds if `nodeFrom` is being written to the `content` content of the object in `nodeTo`.
    */
   predicate storeStep(Node nodeFrom, Node nodeTo, Content content) {
-    exists(DataFlowPublic::AttrWrite a |
-      a.mayHaveAttributeName(content) and
+    exists(DataFlowPublic::AttrWrite a, string attrName |
+      content.(DataFlowPublic::AttributeContent).getAttribute() = attrName and
+      a.mayHaveAttributeName(attrName) and
       nodeFrom = a.getValue() and
       nodeTo = a.getObject()
     )
     or
-    exists(DataFlowPublic::ContentSet contents |
-      contents.(DataFlowPublic::AttributeContent).getAttribute() = content
+    // type-tracking doesn't really handle PostUpdateNodes, so for some assignment steps
+    // like `my_dict["foo"] = foo` the data-flow step targets the PostUpdateNode for
+    // `my_dict`, where we want to translate that into a type-tracking step that targets
+    // the normal/non-PostUpdateNode for `my_dict`.
+    exists(DataFlowPublic::Node storeTarget |
+      DataFlowPrivate::storeStepCommon(nodeFrom, content, storeTarget)
     |
-      TypeTrackerSummaryFlow::basicStoreStep(nodeFrom, nodeTo, contents)
-    )
+      not storeTarget instanceof DataFlowPrivate::SyntheticPostUpdateNode and
+      nodeTo = storeTarget
+      or
+      nodeTo = storeTarget.(DataFlowPrivate::SyntheticPostUpdateNode).getPreUpdateNode()
+    ) and
+    // when only supporting precise content, no need for IterableElementNode (since it
+    // is only fed set/list content)
+    not nodeFrom instanceof DataFlowPublic::IterableElementNode
+    or
+    TypeTrackerSummaryFlow::basicStoreStep(nodeFrom, nodeTo, content)
   }
 
   /**
    * Holds if `nodeTo` is the result of accessing the `content` content of `nodeFrom`.
    */
   predicate loadStep(Node nodeFrom, LocalSourceNode nodeTo, Content content) {
-    exists(DataFlowPublic::AttrRead a |
-      a.mayHaveAttributeName(content) and
+    exists(DataFlowPublic::AttrRead a, string attrName |
+      content.(DataFlowPublic::AttributeContent).getAttribute() = attrName and
+      a.mayHaveAttributeName(attrName) and
       nodeFrom = a.getObject() and
       nodeTo = a
     )
     or
-    exists(DataFlowPublic::ContentSet contents |
-      contents.(DataFlowPublic::AttributeContent).getAttribute() = content
-    |
-      TypeTrackerSummaryFlow::basicLoadStep(nodeFrom, nodeTo, contents)
+    DataFlowPrivate::readStepCommon(nodeFrom, content, nodeTo) and
+    // Since we only support one level of content in type-trackers we don't actually
+    // support `(aa, ab), (ba, bb) = ...`. Therefore we exclude the read-step from `(aa,
+    // ab)` to `aa` (since it is not needed).
+    not exists(SequenceNode outer |
+      outer.getAnElement() = nodeFrom.asCfgNode() and
+      IterableUnpacking::iterableUnpackingTupleFlowStep(_, nodeFrom)
+    ) and
+    // Again, due to only supporting one level deep, for `for (k,v) in ...` we exclude read-step from
+    // the tuple to `k` and `v`.
+    not exists(DataFlowPublic::IterableSequenceNode seq, DataFlowPublic::IterableElementNode elem |
+      IterableUnpacking::iterableUnpackingForReadStep(_, _, seq) and
+      IterableUnpacking::iterableUnpackingConvertingReadStep(seq, _, elem) and
+      IterableUnpacking::iterableUnpackingConvertingStoreStep(elem, _, nodeFrom) and
+      nodeFrom.asCfgNode() instanceof SequenceNode
     )
+    or
+    TypeTrackerSummaryFlow::basicLoadStep(nodeFrom, nodeTo, content)
   }
 
   /**
    * Holds if the `loadContent` of `nodeFrom` is stored in the `storeContent` of `nodeTo`.
    */
   predicate loadStoreStep(Node nodeFrom, Node nodeTo, Content loadContent, Content storeContent) {
-    exists(DataFlowPublic::ContentSet loadContents, DataFlowPublic::ContentSet storeContents |
-      loadContents.(DataFlowPublic::AttributeContent).getAttribute() = loadContent and
-      storeContents.(DataFlowPublic::AttributeContent).getAttribute() = storeContent
-    |
-      TypeTrackerSummaryFlow::basicLoadStoreStep(nodeFrom, nodeTo, loadContents, storeContents)
-    )
+    TypeTrackerSummaryFlow::basicLoadStoreStep(nodeFrom, nodeTo, loadContent, storeContent)
   }
 
   /**

python/ql/lib/semmle/python/dataflow/new/internal/tainttracking1/TaintTrackingImpl.qll

@@ -127,7 +127,7 @@ abstract deprecated class Configuration extends DataFlow::Configuration {
 
   final override predicate isAdditionalFlowStep(DataFlow::Node node1, DataFlow::Node node2) {
     this.isAdditionalTaintStep(node1, node2) or
-    defaultAdditionalTaintStep(node1, node2)
+    defaultAdditionalTaintStep(node1, node2, _)
   }
 
   /**

python/ql/lib/semmle/python/dataflow/new/internal/tainttracking2/TaintTrackingImpl.qll

@@ -127,7 +127,7 @@ abstract deprecated class Configuration extends DataFlow::Configuration {
 
   final override predicate isAdditionalFlowStep(DataFlow::Node node1, DataFlow::Node node2) {
     this.isAdditionalTaintStep(node1, node2) or
-    defaultAdditionalTaintStep(node1, node2)
+    defaultAdditionalTaintStep(node1, node2, _)
   }
 
   /**

python/ql/lib/semmle/python/dataflow/new/internal/tainttracking3/TaintTrackingImpl.qll

@@ -127,7 +127,7 @@ abstract deprecated class Configuration extends DataFlow::Configuration {
 
   final override predicate isAdditionalFlowStep(DataFlow::Node node1, DataFlow::Node node2) {
     this.isAdditionalTaintStep(node1, node2) or
-    defaultAdditionalTaintStep(node1, node2)
+    defaultAdditionalTaintStep(node1, node2, _)
   }
 
   /**

python/ql/lib/semmle/python/dataflow/new/internal/tainttracking4/TaintTrackingImpl.qll

@@ -127,7 +127,7 @@ abstract deprecated class Configuration extends DataFlow::Configuration {
 
   final override predicate isAdditionalFlowStep(DataFlow::Node node1, DataFlow::Node node2) {
     this.isAdditionalTaintStep(node1, node2) or
-    defaultAdditionalTaintStep(node1, node2)
+    defaultAdditionalTaintStep(node1, node2, _)
   }
 
   /**

python/ql/lib/semmle/python/dataflow/old/Implementation.qll

@@ -410,7 +410,7 @@ class TaintTrackingImplementation extends string instanceof TaintTracking::Confi
         call = node.asCfgNode() and
         call.getFunction().pointsTo(ObjectInternal::builtin("getattr")) and
         arg = call.getArg(0) and
-        attrname = call.getArg(1).getNode().(StrConst).getText() and
+        attrname = call.getArg(1).getNode().(StringLiteral).getText() and
         arg = srcnode.asCfgNode()
       |
         path = srcpath.fromAttribute(attrname) and

python/ql/lib/semmle/python/frameworks/Asyncpg.model.yml

@@ -0,0 +1,29 @@
+extensions:
+  - addsTo:
+      pack: codeql/python-all
+      extensible: sinkModel
+    data:
+      # `Connection`s and `ConnectionPool`s provide some methods that execute SQL.
+      - ['asyncpg.~Connection', 'Member[copy_from_query,execute,fetch,fetchrow,fetchval].Argument[0,query:]', 'sql-injection']
+      - ['asyncpg.~Connection', 'Member[executemany].Argument[0,command:]', 'sql-injection']
+      # A model of `Connection` and `ConnectionPool`, which provide some methods that access the file system.
+      - ['asyncpg.~Connection', 'Member[copy_from_query,copy_from_table].Argument[output:]', 'path-injection']
+      - ['asyncpg.~Connection', 'Member[copy_to_table].Argument[source:]', 'path-injection']
+      # the `PreparedStatement` class in `asyncpg`.
+      - ['asyncpg.Connection', 'Member[prepare].Argument[0,query:]', 'sql-injection']
+
+  - addsTo:
+      pack: codeql/python-all
+      extensible: typeModel
+    data:
+      # a `ConnectionPool` that is created when the result of `asyncpg.create_pool()` is awaited.
+      - ['asyncpg.Connection', 'asyncpg.ConnectionPool', 'Member[acquire].ReturnValue.Awaited']
+      # a `Connection` that is created when
+      # * - the result of `asyncpg.connect()` is awaited.
+      # * - the result of calling `acquire` on a `ConnectionPool` is awaited.
+      - ['asyncpg.Connection', 'asyncpg', 'Member[connect].ReturnValue.Awaited']
+      - ['asyncpg.Connection', 'asyncpg', 'Member[connection].Member[connect].ReturnValue.Awaited']
+      - ['asyncpg.ConnectionPool', 'asyncpg', 'Member[create_pool].ReturnValue.Awaited']
+      # Creating an internal `~Connection` type that contains both `Connection` and `ConnectionPool`.
+      - ['asyncpg.~Connection', 'asyncpg.Connection', '']
+      - ['asyncpg.~Connection', 'asyncpg.ConnectionPool', '']

python/ql/lib/semmle/python/frameworks/Asyncpg.qll

@@ -11,43 +11,6 @@ private import semmle.python.frameworks.data.ModelsAsData
 
 /** Provides models for the `asyncpg` PyPI package. */
 private module Asyncpg {
-  class AsyncpgModel extends ModelInput::TypeModelCsv {
-    override predicate row(string row) {
-      // type1;type2;path
-      row =
-        [
-          // a `ConnectionPool` that is created when the result of `asyncpg.create_pool()` is awaited.
-          "asyncpg.ConnectionPool;asyncpg;Member[create_pool].ReturnValue.Awaited",
-          // a `Connection` that is created when
-          // * - the result of `asyncpg.connect()` is awaited.
-          // * - the result of calling `acquire` on a `ConnectionPool` is awaited.
-          "asyncpg.Connection;asyncpg;Member[connect].ReturnValue.Awaited",
-          "asyncpg.Connection;asyncpg;Member[connection].Member[connect].ReturnValue.Awaited",
-          "asyncpg.Connection;asyncpg.ConnectionPool;Member[acquire].ReturnValue.Awaited",
-          // Creating an internal `~Connection` type that contains both `Connection` and `ConnectionPool`.
-          "asyncpg.~Connection;asyncpg.Connection;", //
-          "asyncpg.~Connection;asyncpg.ConnectionPool;"
-        ]
-    }
-  }
-
-  class AsyncpgSink extends ModelInput::SinkModelCsv {
-    // type;path;kind
-    override predicate row(string row) {
-      row =
-        [
-          // `Connection`s and `ConnectionPool`s provide some methods that execute SQL.
-          "asyncpg.~Connection;Member[copy_from_query,execute,fetch,fetchrow,fetchval].Argument[0,query:];sql-injection",
-          "asyncpg.~Connection;Member[executemany].Argument[0,command:];sql-injection",
-          // A model of `Connection` and `ConnectionPool`, which provide some methods that access the file system.
-          "asyncpg.~Connection;Member[copy_from_query,copy_from_table].Argument[output:];path-injection",
-          "asyncpg.~Connection;Member[copy_to_table].Argument[source:];path-injection",
-          // the `PreparedStatement` class in `asyncpg`.
-          "asyncpg.Connection;Member[prepare].Argument[0,query:];sql-injection",
-        ]
-    }
-  }
-
   /**
    * Provides models of the `Cursor` class in `asyncpg`.
    * `Cursor`s are created

python/ql/lib/semmle/python/frameworks/Cryptodome.qll

@@ -83,7 +83,7 @@ private module CryptodomeModel {
 
     /** Gets the name of the curve to use, as well as the origin that explains how we obtained this name. */
     string getCurveWithOrigin(DataFlow::Node origin) {
-      exists(StrConst str | origin = DataFlow::exprNode(str) |
+      exists(StringLiteral str | origin = DataFlow::exprNode(str) |
         origin = this.getCurveArg().getALocalSource() and
         result = str.getText()
       )

python/ql/lib/semmle/python/frameworks/Django.qll

@@ -2862,14 +2862,14 @@ module PrivateDjango {
         //
         // This also strongly implies that `mw` is in fact a Django middleware setting and
         // not just a variable named `MIDDLEWARE`.
-        list.getAnElt().(StrConst).getText() =
+        list.getAnElt().(StringLiteral).getText() =
           "django.contrib.auth.middleware.AuthenticationMiddleware"
       )
     }
 
     override boolean getVerificationSetting() {
       if
-        list.getAnElt().(StrConst).getText() in [
+        list.getAnElt().(StringLiteral).getText() in [
             "django.middleware.csrf.CsrfViewMiddleware",
             // see https://github.com/mozilla/django-session-csrf
             "session_csrf.CsrfMiddleware"

python/ql/lib/semmle/python/frameworks/FastApi.qll

@@ -183,7 +183,7 @@ module FastApi {
       |
         exists(Assign assign | assign = cls.getAStmt() |
           assign.getATarget().(Name).getId() = "media_type" and
-          result = assign.getValue().(StrConst).getText()
+          result = assign.getValue().(StringLiteral).getText()
         )
         or
         // TODO: this should use a proper MRO calculation instead
@@ -372,7 +372,7 @@ module FastApi {
           headers.accesses(instance(), "headers") and
           this.calls(headers, "append") and
           keyArg in [this.getArg(0), this.getArgByName("key")] and
-          keyArg.getALocalSource().asExpr().(StrConst).getText().toLowerCase() = "set-cookie"
+          keyArg.getALocalSource().asExpr().(StringLiteral).getText().toLowerCase() = "set-cookie"
         )
       }
 

python/ql/lib/semmle/python/frameworks/Rsa.qll

@@ -80,7 +80,7 @@ private module Rsa {
       result.getName() = "RSA"
       or
       // hashing part
-      exists(StrConst str, DataFlow::Node hashNameArg |
+      exists(StringLiteral str, DataFlow::Node hashNameArg |
         hashNameArg in [this.getArg(2), this.getArgByName("hash_method")] and
         DataFlow::exprNode(str) = hashNameArg.getALocalSource() and
         result.matchesName(str.getText())
@@ -132,7 +132,7 @@ private module Rsa {
     override DataFlow::Node getInitialization() { result = this }
 
     override Cryptography::CryptographicAlgorithm getAlgorithm() {
-      exists(StrConst str, DataFlow::Node hashNameArg |
+      exists(StringLiteral str, DataFlow::Node hashNameArg |
         hashNameArg in [this.getArg(1), this.getArgByName("method_name")] and
         DataFlow::exprNode(str) = hashNameArg.getALocalSource() and
         result.matchesName(str.getText())

python/ql/lib/semmle/python/frameworks/ServerLess.qll

@@ -8,7 +8,7 @@
  */
 
 import python
-import codeql.serverless.ServerLess
+private import codeql.serverless.ServerLess
 import semmle.python.dataflow.new.DataFlow
 import semmle.python.dataflow.new.RemoteFlowSources
 

python/ql/lib/semmle/python/frameworks/Stdlib.qll

@@ -2785,7 +2785,7 @@ module StdlibPrivate {
   /** Gets a call to `hashlib.new` with `algorithmName` as the first argument. */
   private API::CallNode hashlibNewCall(string algorithmName) {
     algorithmName =
-      result.getParameter(0, "name").getAValueReachingSink().asExpr().(StrConst).getText() and
+      result.getParameter(0, "name").getAValueReachingSink().asExpr().(StringLiteral).getText() and
     result = API::moduleImport("hashlib").getMember("new").getACall()
   }
 
@@ -2908,7 +2908,8 @@ module StdlibPrivate {
       exists(string algorithmName | result.matchesName(algorithmName) |
         this.getDigestArg().asSink() = hashlibMember(algorithmName).asSource()
         or
-        this.getDigestArg().getAValueReachingSink().asExpr().(StrConst).getText() = algorithmName
+        this.getDigestArg().getAValueReachingSink().asExpr().(StringLiteral).getText() =
+          algorithmName
       )
     }
 
@@ -4418,7 +4419,7 @@ module StdlibPrivate {
 
     override DataFlow::CallCfgNode getACall() {
       result.(DataFlow::MethodCallNode).getMethodName() = "pop" and
-      result.getArg(0).getALocalSource().asExpr().(StrConst).getText() = key
+      result.getArg(0).getALocalSource().asExpr().(StringLiteral).getText() = key
     }
 
     override DataFlow::ArgumentNode getACallback() { none() }
@@ -4441,7 +4442,7 @@ module StdlibPrivate {
 
     override DataFlow::CallCfgNode getACall() {
       result.(DataFlow::MethodCallNode).getMethodName() = "get" and
-      result.getArg(0).getALocalSource().asExpr().(StrConst).getText() = key
+      result.getArg(0).getALocalSource().asExpr().(StringLiteral).getText() = key
     }
 
     override DataFlow::ArgumentNode getACallback() { none() }
@@ -4541,7 +4542,7 @@ module StdlibPrivate {
 
     override DataFlow::CallCfgNode getACall() {
       result.(DataFlow::MethodCallNode).getMethodName() = "setdefault" and
-      result.getArg(0).getALocalSource().asExpr().(StrConst).getText() = key
+      result.getArg(0).getALocalSource().asExpr().(StringLiteral).getText() = key
     }
 
     override DataFlow::ArgumentNode getACallback() { none() }

python/ql/lib/semmle/python/frameworks/Urllib3.qll

@@ -78,7 +78,7 @@ module Urllib3 {
           // see https://urllib3.readthedocs.io/en/stable/user-guide.html?highlight=cert_reqs#certificate-verification
           disablingNode = constructor.getKeywordParameter("cert_reqs").asSink() and
           argumentOrigin = constructor.getKeywordParameter("cert_reqs").getAValueReachingSink() and
-          argumentOrigin.asExpr().(StrConst).getText() = "CERT_NONE"
+          argumentOrigin.asExpr().(StringLiteral).getText() = "CERT_NONE"
           or
           // assert_hostname
           // see https://urllib3.readthedocs.io/en/stable/reference/urllib3.connectionpool.html?highlight=assert_hostname#urllib3.HTTPSConnectionPool

python/ql/lib/semmle/python/frameworks/data/ModelsAsData.qll

@@ -22,7 +22,7 @@ private import semmle.python.dataflow.new.FlowSummary
 /**
  * A remote flow source originating from a CSV source row.
  */
-private class RemoteFlowSourceFromCsv extends RemoteFlowSource {
+private class RemoteFlowSourceFromCsv extends RemoteFlowSource::Range {
   RemoteFlowSourceFromCsv() { this = ModelOutput::getASourceNode("remote").asSource() }
 
   override string getSourceType() { result = "Remote flow (from model)" }
@@ -33,7 +33,7 @@ private class SummarizedCallableFromModel extends SummarizedCallable {
   string path;
 
   SummarizedCallableFromModel() {
-    ModelOutput::relevantSummaryModel(type, path, _, _, _) and
+    ModelOutput::relevantSummaryModel(type, path, _, _, _, _) and
     this = type + ";" + path
   }
 
@@ -46,8 +46,10 @@ private class SummarizedCallableFromModel extends SummarizedCallable {
     )
   }
 
-  override predicate propagatesFlow(string input, string output, boolean preservesValue) {
-    exists(string kind | ModelOutput::relevantSummaryModel(type, path, input, output, kind) |
+  override predicate propagatesFlow(
+    string input, string output, boolean preservesValue, string model
+  ) {
+    exists(string kind | ModelOutput::relevantSummaryModel(type, path, input, output, kind, model) |
       kind = "value" and
       preservesValue = true
       or

python/ql/lib/semmle/python/frameworks/data/internal/ApiGraphModels.qll

@@ -1,17 +1,17 @@
 /**
  * INTERNAL use only. This is an experimental API subject to change without notice.
  *
- * Provides classes and predicates for dealing with flow models specified in CSV format.
+ * Provides classes and predicates for dealing with flow models specified in extensible predicates.
  *
- * The CSV specification has the following columns:
+ * The extensible predicates have the following columns:
  * - Sources:
- *   `type; path; kind`
+ *   `type, path, kind`
  * - Sinks:
- *   `type; path; kind`
+ *   `type, path, kind`
  * - Summaries:
- *   `type; path; input; output; kind`
+ *   `type, path, input, output, kind`
  * - Types:
- *   `type1; type2; path`
+ *   `type1, type2, path`
  *
  * The interpretation of a row is similar to API-graphs with a left-to-right
  * reading.
@@ -76,11 +76,13 @@ private import codeql.dataflow.internal.AccessPathSyntax
 /** Module containing hooks for providing input data to be interpreted as a model. */
 module ModelInput {
   /**
+   * DEPRECATED: Use the extensible predicate `sourceModel` instead.
+   *
    * A unit class for adding additional source model rows.
    *
    * Extend this class to add additional source definitions.
    */
-  class SourceModelCsv extends Unit {
+  deprecated class SourceModelCsv extends Unit {
     /**
      * Holds if `row` specifies a source definition.
      *
@@ -93,15 +95,17 @@ module ModelInput {
      *
      * The kind `remote` represents a general remote flow source.
      */
-    abstract predicate row(string row);
+    abstract deprecated predicate row(string row);
   }
 
   /**
    * A unit class for adding additional sink model rows.
    *
    * Extend this class to add additional sink definitions.
+   *
+   * DEPRECATED: Use the extensible predicate `sinkModel` instead.
    */
-  class SinkModelCsv extends Unit {
+  deprecated class SinkModelCsv extends Unit {
     /**
      * Holds if `row` specifies a sink definition.
      *
@@ -112,15 +116,17 @@ module ModelInput {
      * indicates that the value at `(type, path)` should be seen as a sink
      * of the given `kind`.
      */
-    abstract predicate row(string row);
+    abstract deprecated predicate row(string row);
   }
 
   /**
    * A unit class for adding additional summary model rows.
    *
    * Extend this class to add additional flow summary definitions.
+   *
+   * DEPRECATED: Use the extensible predicate `summaryModel` instead.
    */
-  class SummaryModelCsv extends Unit {
+  deprecated class SummaryModelCsv extends Unit {
     /**
      * Holds if `row` specifies a summary definition.
      *
@@ -134,15 +140,18 @@ module ModelInput {
      * `kind` should be either `value` or `taint`, for value-preserving or taint-preserving steps,
      * respectively.
      */
-    abstract predicate row(string row);
+    abstract deprecated predicate row(string row);
   }
 
   /**
    * A unit class for adding additional type model rows.
    *
    * Extend this class to add additional type definitions.
+   *
+   * DEPRECATED: Use the extensible predicate `typeModel` or the class
+   * `TypeModel` instead.
    */
-  class TypeModelCsv extends Unit {
+  deprecated class TypeModelCsv extends Unit {
     /**
      * Holds if `row` specifies a type definition.
      *
@@ -152,7 +161,7 @@ module ModelInput {
      * ```
      * indicates that `(type2, path)` should be seen as an instance of `type1`.
      */
-    abstract predicate row(string row);
+    abstract deprecated predicate row(string row);
   }
 
   /**
@@ -186,8 +195,10 @@ module ModelInput {
 
   /**
    * A unit class for adding additional type variable model rows.
+   *
+   * DEPRECATED: Use the extensible predicate `typeVariableModel` instead.
    */
-  class TypeVariableModelCsv extends Unit {
+  deprecated class TypeVariableModelCsv extends Unit {
     /**
      * Holds if `row` specifies a path through a type variable.
      *
@@ -197,7 +208,7 @@ module ModelInput {
      * ```
      * means `path` can be substituted for a token `TypeVar[name]`.
      */
-    abstract predicate row(string row);
+    abstract deprecated predicate row(string row);
   }
 }
 
@@ -216,87 +227,141 @@ abstract class TestAllModels extends Unit { }
  * does not preserve empty trailing substrings.
  */
 bindingset[result]
-private string inversePad(string s) { s = result + ";dummy" }
+deprecated private string inversePad(string s) { s = result + ";dummy" }
 
-private predicate sourceModel(string row) { any(SourceModelCsv s).row(inversePad(row)) }
+deprecated private predicate sourceModel(string row) { any(SourceModelCsv s).row(inversePad(row)) }
 
-private predicate sinkModel(string row) { any(SinkModelCsv s).row(inversePad(row)) }
+deprecated private predicate sinkModel(string row) { any(SinkModelCsv s).row(inversePad(row)) }
 
-private predicate summaryModel(string row) { any(SummaryModelCsv s).row(inversePad(row)) }
+deprecated private predicate summaryModel(string row) {
+  any(SummaryModelCsv s).row(inversePad(row))
+}
 
-private predicate typeModel(string row) { any(TypeModelCsv s).row(inversePad(row)) }
+deprecated private predicate typeModel(string row) { any(TypeModelCsv s).row(inversePad(row)) }
 
-private predicate typeVariableModel(string row) { any(TypeVariableModelCsv s).row(inversePad(row)) }
+deprecated private predicate typeVariableModel(string row) {
+  any(TypeVariableModelCsv s).row(inversePad(row))
+}
+
+private class DeprecationAdapter extends Unit {
+  abstract predicate sourceModel(string type, string path, string kind);
+
+  abstract predicate sinkModel(string type, string path, string kind);
+
+  abstract predicate summaryModel(string type, string path, string input, string output, string kind);
+
+  abstract predicate typeModel(string type1, string type2, string path);
+
+  abstract predicate typeVariableModel(string name, string path);
+}
+
+private class DeprecationAdapterImpl extends DeprecationAdapter {
+  deprecated override predicate sourceModel(string type, string path, string kind) {
+    exists(string row |
+      sourceModel(row) and
+      row.splitAt(";", 0) = type and
+      row.splitAt(";", 1) = path and
+      row.splitAt(";", 2) = kind
+    )
+  }
+
+  deprecated override predicate sinkModel(string type, string path, string kind) {
+    exists(string row |
+      sinkModel(row) and
+      row.splitAt(";", 0) = type and
+      row.splitAt(";", 1) = path and
+      row.splitAt(";", 2) = kind
+    )
+  }
+
+  deprecated override predicate summaryModel(
+    string type, string path, string input, string output, string kind
+  ) {
+    exists(string row |
+      summaryModel(row) and
+      row.splitAt(";", 0) = type and
+      row.splitAt(";", 1) = path and
+      row.splitAt(";", 2) = input and
+      row.splitAt(";", 3) = output and
+      row.splitAt(";", 4) = kind
+    )
+  }
+
+  deprecated override predicate typeModel(string type1, string type2, string path) {
+    exists(string row |
+      typeModel(row) and
+      row.splitAt(";", 0) = type1 and
+      row.splitAt(";", 1) = type2 and
+      row.splitAt(";", 2) = path
+    )
+  }
+
+  deprecated override predicate typeVariableModel(string name, string path) {
+    exists(string row |
+      typeVariableModel(row) and
+      row.splitAt(";", 0) = name and
+      row.splitAt(";", 1) = path
+    )
+  }
+}
 
 /** Holds if a source model exists for the given parameters. */
-predicate sourceModel(string type, string path, string kind) {
-  exists(string row |
-    sourceModel(row) and
-    row.splitAt(";", 0) = type and
-    row.splitAt(";", 1) = path and
-    row.splitAt(";", 2) = kind
-  )
+predicate sourceModel(string type, string path, string kind, string model) {
+  any(DeprecationAdapter a).sourceModel(type, path, kind) and
+  model = "SourceModelCsv"
   or
-  Extensions::sourceModel(type, path, kind)
+  exists(QlBuiltins::ExtensionId madId |
+    Extensions::sourceModel(type, path, kind, madId) and
+    model = "MaD:" + madId.toString()
+  )
 }
 
 /** Holds if a sink model exists for the given parameters. */
-private predicate sinkModel(string type, string path, string kind) {
-  exists(string row |
-    sinkModel(row) and
-    row.splitAt(";", 0) = type and
-    row.splitAt(";", 1) = path and
-    row.splitAt(";", 2) = kind
-  )
+private predicate sinkModel(string type, string path, string kind, string model) {
+  any(DeprecationAdapter a).sinkModel(type, path, kind) and
+  model = "SinkModelCsv"
   or
-  Extensions::sinkModel(type, path, kind)
+  exists(QlBuiltins::ExtensionId madId |
+    Extensions::sinkModel(type, path, kind, madId) and
+    model = "MaD:" + madId.toString()
+  )
 }
 
 /** Holds if a summary model `row` exists for the given parameters. */
-private predicate summaryModel(string type, string path, string input, string output, string kind) {
-  exists(string row |
-    summaryModel(row) and
-    row.splitAt(";", 0) = type and
-    row.splitAt(";", 1) = path and
-    row.splitAt(";", 2) = input and
-    row.splitAt(";", 3) = output and
-    row.splitAt(";", 4) = kind
-  )
+private predicate summaryModel(
+  string type, string path, string input, string output, string kind, string model
+) {
+  any(DeprecationAdapter a).summaryModel(type, path, input, output, kind) and
+  model = "SummaryModelCsv"
   or
-  Extensions::summaryModel(type, path, input, output, kind)
+  exists(QlBuiltins::ExtensionId madId |
+    Extensions::summaryModel(type, path, input, output, kind, madId) and
+    model = "MaD:" + madId.toString()
+  )
 }
 
 /** Holds if a type model exists for the given parameters. */
-private predicate typeModel(string type1, string type2, string path) {
-  exists(string row |
-    typeModel(row) and
-    row.splitAt(";", 0) = type1 and
-    row.splitAt(";", 1) = type2 and
-    row.splitAt(";", 2) = path
-  )
+predicate typeModel(string type1, string type2, string path) {
+  any(DeprecationAdapter a).typeModel(type1, type2, path)
   or
   Extensions::typeModel(type1, type2, path)
 }
 
 /** Holds if a type variable model exists for the given parameters. */
 private predicate typeVariableModel(string name, string path) {
-  exists(string row |
-    typeVariableModel(row) and
-    row.splitAt(";", 0) = name and
-    row.splitAt(";", 1) = path
-  )
+  any(DeprecationAdapter a).typeVariableModel(name, path)
   or
   Extensions::typeVariableModel(name, path)
 }
 
 /**
- * Holds if CSV rows involving `type` might be relevant for the analysis of this database.
+ * Holds if rows involving `type` might be relevant for the analysis of this database.
  */
 predicate isRelevantType(string type) {
   (
-    sourceModel(type, _, _) or
-    sinkModel(type, _, _) or
-    summaryModel(type, _, _, _, _) or
+    sourceModel(type, _, _, _) or
+    sinkModel(type, _, _, _) or
+    summaryModel(type, _, _, _, _, _) or
     typeModel(_, type, _)
   ) and
   (
@@ -313,26 +378,26 @@ predicate isRelevantType(string type) {
 }
 
 /**
- * Holds if `type,path` is used in some CSV row.
+ * Holds if `type,path` is used in some row.
  */
 pragma[nomagic]
 predicate isRelevantFullPath(string type, string path) {
   isRelevantType(type) and
   (
-    sourceModel(type, path, _) or
-    sinkModel(type, path, _) or
-    summaryModel(type, path, _, _, _) or
+    sourceModel(type, path, _, _) or
+    sinkModel(type, path, _, _) or
+    summaryModel(type, path, _, _, _, _) or
     typeModel(_, type, path)
   )
 }
 
-/** A string from a CSV row that should be parsed as an access path. */
+/** A string from a row that should be parsed as an access path. */
 private predicate accessPathRange(string s) {
   isRelevantFullPath(_, s)
   or
   exists(string type | isRelevantType(type) |
-    summaryModel(type, _, s, _, _) or
-    summaryModel(type, _, _, s, _)
+    summaryModel(type, _, s, _, _, _) or
+    summaryModel(type, _, _, s, _, _)
   )
   or
   typeVariableModel(_, s)
@@ -435,7 +500,7 @@ private API::Node getNodeFromType(string type) {
  * Gets the API node identified by the first `n` tokens of `path` in the given `(type, path)` tuple.
  */
 pragma[nomagic]
-private API::Node getNodeFromPath(string type, AccessPath path, int n) {
+API::Node getNodeFromPath(string type, AccessPath path, int n) {
   isRelevantFullPath(type, path) and
   (
     n = 0 and
@@ -543,7 +608,7 @@ private API::Node getNodeFromPath(string type, AccessPath path) {
 
 pragma[nomagic]
 private predicate typeStepModel(string type, AccessPath basePath, AccessPath output) {
-  summaryModel(type, basePath, "", output, "type")
+  summaryModel(type, basePath, "", output, "type", _)
 }
 
 pragma[nomagic]
@@ -618,36 +683,36 @@ module ModelOutput {
   cached
   private module Cached {
     /**
-     * Holds if a CSV source model contributed `source` with the given `kind`.
+     * Holds if a source model contributed `source` with the given `kind`.
      */
     cached
-    API::Node getASourceNode(string kind) {
+    API::Node getASourceNode(string kind, string model) {
       exists(string type, string path |
-        sourceModel(type, path, kind) and
+        sourceModel(type, path, kind, model) and
         result = getNodeFromPath(type, path)
       )
     }
 
     /**
-     * Holds if a CSV sink model contributed `sink` with the given `kind`.
+     * Holds if a sink model contributed `sink` with the given `kind`.
      */
     cached
-    API::Node getASinkNode(string kind) {
+    API::Node getASinkNode(string kind, string model) {
       exists(string type, string path |
-        sinkModel(type, path, kind) and
+        sinkModel(type, path, kind, model) and
         result = getNodeFromPath(type, path)
       )
     }
 
     /**
-     * Holds if a relevant CSV summary exists for these parameters.
+     * Holds if a relevant summary exists for these parameters.
      */
     cached
     predicate relevantSummaryModel(
-      string type, string path, string input, string output, string kind
+      string type, string path, string input, string output, string kind, string model
     ) {
       isRelevantType(type) and
-      summaryModel(type, path, input, output, kind)
+      summaryModel(type, path, input, output, kind, model)
     }
 
     /**
@@ -655,7 +720,7 @@ module ModelOutput {
      */
     cached
     predicate resolvedSummaryBase(string type, string path, Specific::InvokeNode baseNode) {
-      summaryModel(type, path, _, _, _) and
+      summaryModel(type, path, _, _, _, _) and
       baseNode = getInvocationFromPath(type, path)
     }
 
@@ -664,13 +729,13 @@ module ModelOutput {
      */
     cached
     predicate resolvedSummaryRefBase(string type, string path, API::Node baseNode) {
-      summaryModel(type, path, _, _, _) and
+      summaryModel(type, path, _, _, _, _) and
       baseNode = getNodeFromPath(type, path)
     }
 
     /**
      * Holds if `node` is seen as an instance of `type` due to a type definition
-     * contributed by a CSV model.
+     * contributed by a model.
      */
     cached
     API::Node getATypeNode(string type) { result = getNodeFromType(type) }
@@ -680,12 +745,22 @@ module ModelOutput {
   import Specific::ModelOutputSpecific
   private import codeql.mad.ModelValidation as SharedModelVal
 
+  /**
+   * Holds if a CSV source model contributed `source` with the given `kind`.
+   */
+  API::Node getASourceNode(string kind) { result = getASourceNode(kind, _) }
+
+  /**
+   * Holds if a CSV sink model contributed `sink` with the given `kind`.
+   */
+  API::Node getASinkNode(string kind) { result = getASinkNode(kind, _) }
+
   private module KindValConfig implements SharedModelVal::KindValidationConfigSig {
-    predicate summaryKind(string kind) { summaryModel(_, _, _, _, kind) }
+    predicate summaryKind(string kind) { summaryModel(_, _, _, _, kind, _) }
 
-    predicate sinkKind(string kind) { sinkModel(_, _, kind) }
+    predicate sinkKind(string kind) { sinkModel(_, _, kind, _) }
 
-    predicate sourceKind(string kind) { sourceModel(_, _, kind) }
+    predicate sourceKind(string kind) { sourceModel(_, _, kind, _) }
   }
 
   private module KindVal = SharedModelVal::KindValidation<KindValConfig>;
@@ -694,25 +769,6 @@ module ModelOutput {
    * Gets an error message relating to an invalid CSV row in a model.
    */
   string getAWarning() {
-    // Check number of columns
-    exists(string row, string kind, int expectedArity, int actualArity |
-      any(SourceModelCsv csv).row(row) and kind = "source" and expectedArity = 3
-      or
-      any(SinkModelCsv csv).row(row) and kind = "sink" and expectedArity = 3
-      or
-      any(SummaryModelCsv csv).row(row) and kind = "summary" and expectedArity = 5
-      or
-      any(TypeModelCsv csv).row(row) and kind = "type" and expectedArity = 3
-      or
-      any(TypeVariableModelCsv csv).row(row) and kind = "type-variable" and expectedArity = 2
-    |
-      actualArity = count(row.indexOf(";")) + 1 and
-      actualArity != expectedArity and
-      result =
-        "CSV " + kind + " row should have " + expectedArity + " columns but has " + actualArity +
-          ": " + row
-    )
-    or
     // Check names and arguments of access path tokens
     exists(AccessPath path, AccessPathToken token |
       (isRelevantFullPath(_, path) or typeVariableModel(_, path)) and

python/ql/lib/semmle/python/frameworks/data/internal/ApiGraphModelsExtensions.qll

@@ -8,13 +8,15 @@
  *
  * The kind `remote` represents a general remote flow source.
  */
-extensible predicate sourceModel(string type, string path, string kind);
+extensible predicate sourceModel(
+  string type, string path, string kind, QlBuiltins::ExtensionId madId
+);
 
 /**
  * Holds if the value at `(type, path)` should be seen as a sink
  * of the given `kind`.
  */
-extensible predicate sinkModel(string type, string path, string kind);
+extensible predicate sinkModel(string type, string path, string kind, QlBuiltins::ExtensionId madId);
 
 /**
  * Holds if in calls to `(type, path)`, the value referred to by `input`
@@ -23,7 +25,9 @@ extensible predicate sinkModel(string type, string path, string kind);
  * `kind` should be either `value` or `taint`, for value-preserving or taint-preserving steps,
  * respectively.
  */
-extensible predicate summaryModel(string type, string path, string input, string output, string kind);
+extensible predicate summaryModel(
+  string type, string path, string input, string output, string kind, QlBuiltins::ExtensionId madId
+);
 
 /**
  * Holds if calls to `(type, path)` should be considered neutral. The meaning of this depends on the `kind`.

python/ql/lib/semmle/python/frameworks/data/internal/ApiGraphModelsSpecific.qll

@@ -138,7 +138,7 @@ predicate invocationMatchesExtraCallSiteFilter(API::CallNode invoke, AccessPathT
 pragma[nomagic]
 private predicate relevantInputOutputPath(API::CallNode base, AccessPath inputOrOutput) {
   exists(string type, string input, string output, string path |
-    ModelOutput::relevantSummaryModel(type, path, input, output, _) and
+    ModelOutput::relevantSummaryModel(type, path, input, output, _, _) and
     ModelOutput::resolvedSummaryBase(type, path, base) and
     inputOrOutput = [input, output]
   )
@@ -170,7 +170,7 @@ private API::Node getNodeFromInputOutputPath(API::CallNode baseNode, AccessPath
  */
 predicate summaryStep(API::Node pred, API::Node succ, string kind) {
   exists(string type, string path, API::CallNode base, AccessPath input, AccessPath output |
-    ModelOutput::relevantSummaryModel(type, path, input, output, kind) and
+    ModelOutput::relevantSummaryModel(type, path, input, output, kind, _) and // TODO???
     ModelOutput::resolvedSummaryBase(type, path, base) and
     pred = getNodeFromInputOutputPath(base, input) and
     succ = getNodeFromInputOutputPath(base, output)

python/ql/lib/semmle/python/frameworks/data/internal/empty.model.yml

@@ -1,5 +1,6 @@
 extensions:
-  # Contribute empty data sets to avoid errors about an undefined extensionals
+  # Make sure that the extensible model predicates have at least one definition
+  # to avoid errors about undefined extensionals.
   - addsTo:
       pack: codeql/python-all
       extensible: sourceModel

python/ql/lib/semmle/python/objects/Constants.qll

@@ -239,8 +239,8 @@ class UnicodeObjectInternal extends ConstantObjectInternal, TUnicode {
 
   override predicate introducedAt(ControlFlowNode node, PointsToContext context) {
     context.appliesTo(node) and
-    node.getNode().(StrConst).getText() = this.strValue() and
-    node.getNode().(StrConst).isUnicode()
+    node.getNode().(StringLiteral).getText() = this.strValue() and
+    node.getNode().(StringLiteral).isUnicode()
   }
 
   override ObjectInternal getClass() { result = TBuiltinClassObject(Builtin::special("unicode")) }
@@ -272,8 +272,8 @@ class BytesObjectInternal extends ConstantObjectInternal, TBytes {
 
   override predicate introducedAt(ControlFlowNode node, PointsToContext context) {
     context.appliesTo(node) and
-    node.getNode().(StrConst).getText() = this.strValue() and
-    not node.getNode().(StrConst).isUnicode()
+    node.getNode().(StringLiteral).getText() = this.strValue() and
+    not node.getNode().(StringLiteral).isUnicode()
   }
 
   override ObjectInternal getClass() { result = TBuiltinClassObject(Builtin::special("bytes")) }

python/ql/lib/semmle/python/objects/ObjectAPI.qll

@@ -201,7 +201,7 @@ class ModuleValue extends Value instanceof ModuleObjectInternal {
     (
       not this.getPath().getExtension() = "py"
       or
-      exists(If i, Name name, StrConst main, Cmpop op |
+      exists(If i, Name name, StringLiteral main, Cmpop op |
         i.getScope() = this.getScope() and
         op instanceof Eq and
         i.getTest().(Compare).compares(name, op, main) and

python/ql/lib/semmle/python/objects/TObject.qll

@@ -84,7 +84,7 @@ newtype TObject =
   /** The unicode string `s` */
   TUnicode(string s) {
     // Any string explicitly mentioned in the source code.
-    exists(StrConst str |
+    exists(StringLiteral str |
       s = str.getText() and
       str.isUnicode()
     )
@@ -100,7 +100,7 @@ newtype TObject =
   /** The byte string `s` */
   TBytes(string s) {
     // Any string explicitly mentioned in the source code.
-    exists(StrConst str |
+    exists(StringLiteral str |
       s = str.getText() and
       not str.isUnicode()
     )

python/ql/lib/semmle/python/pointsto/Filters.qll

@@ -9,7 +9,7 @@ import python
 predicate hasattr(CallNode c, ControlFlowNode obj, string attr) {
   c.getFunction().getNode().(Name).getId() = "hasattr" and
   c.getArg(0) = obj and
-  c.getArg(1).getNode().(StrConst).getText() = attr
+  c.getArg(1).getNode().(StringLiteral).getText() = attr
 }
 
 /** Holds if `c` is a call to `isinstance(use, cls)`. */

python/ql/lib/semmle/python/pointsto/PointsTo.qll

@@ -691,7 +691,7 @@ module PointsToInternal {
         sub.getObject() = sys_modules_flow and
         pointsTo(sys_modules_flow, _, ObjectInternal::sysModules(), _) and
         sub.getIndex() = n and
-        n.getNode().(StrConst).getText() = name and
+        n.getNode().(StringLiteral).getText() = name and
         sub.(DefinitionNode).getValue() = mod and
         pointsTo(mod, _, m, _)
       )

python/ql/lib/semmle/python/pointsto/PointsToContext.qll

@@ -253,7 +253,7 @@ predicate executes_in_runtime_context(Function f) {
 }
 
 private predicate maybe_main(Module m) {
-  exists(If i, Compare cmp, Name name, StrConst main | m.getAStmt() = i and i.getTest() = cmp |
+  exists(If i, Compare cmp, Name name, StringLiteral main | m.getAStmt() = i and i.getTest() = cmp |
     cmp.compares(name, any(Eq eq), main) and
     name.getId() = "__name__" and
     main.getText() = "__main__"

python/ql/lib/semmle/python/regex.qll

@@ -15,7 +15,7 @@ RegExpTerm getTermForExecution(Concepts::RegexExecution exec) {
   )
 }
 
-/** A StrConst used as a regular expression */
+/** A StringLiteral used as a regular expression */
 deprecated class RegexString extends Regex {
   RegexString() { this = RegExpTracking::regExpSource(_).asExpr() }
 }

python/ql/lib/semmle/python/regexp/RegexTreeView.qll

@@ -9,7 +9,7 @@ import Impl as RegexTreeView
 import Impl
 
 /** Gets the parse tree resulting from parsing `re`, if such has been constructed. */
-RegExpTerm getParsedRegExp(StrConst re) { result.getRegex() = re and result.isRootTerm() }
+RegExpTerm getParsedRegExp(StringLiteral re) { result.getRegex() = re and result.isRootTerm() }
 
 /**
  * An element containing a regular expression term, that is, either
@@ -230,7 +230,8 @@ module Impl implements RegexTreeViewSig {
       index > 0 and
       exists(int previousOffset | previousOffset = this.getPartOffset(index - 1) |
         result =
-          previousOffset + re.(StrConst).getImplicitlyConcatenatedPart(index - 1).getContentLength()
+          previousOffset +
+            re.(StringLiteral).getImplicitlyConcatenatedPart(index - 1).getContentLength()
       )
     }
 
@@ -240,7 +241,7 @@ module Impl implements RegexTreeViewSig {
      */
     StringPart getPart(int localOffset) {
       exists(int index, int prefixLength | index = max(int i | this.getPartOffset(i) <= start) |
-        result = re.(StrConst).getImplicitlyConcatenatedPart(index) and
+        result = re.(StringLiteral).getImplicitlyConcatenatedPart(index) and
         result.contextSize(prefixLength, _) and
         // Example:
         // re.compile('...' r"""...this..""")

python/ql/lib/semmle/python/regexp/internal/ParseRegExp.qll

@@ -105,8 +105,8 @@ private module FindRegexMode {
  */
 deprecated class Regex = RegExp;
 
-/** A StrConst used as a regular expression */
-class RegExp extends Expr instanceof StrConst {
+/** A StringLiteral used as a regular expression */
+class RegExp extends Expr instanceof StringLiteral {
   DataFlow::Node use;
 
   RegExp() { this = RegExpTracking::regExpSource(use).asExpr() }

python/ql/lib/semmle/python/regexp/internal/RegExpTracking.qll

@@ -15,7 +15,7 @@ private import semmle.python.dataflow.new.DataFlow
 private import semmle.python.Concepts as Concepts
 
 /** Gets a constant string value that may be used as a regular expression. */
-DataFlow::LocalSourceNode strStart() { result.asExpr() instanceof StrConst }
+DataFlow::LocalSourceNode strStart() { result.asExpr() instanceof StringLiteral }
 
 private import semmle.python.regex as Regex
 

python/ql/lib/semmle/python/security/dataflow/LogInjectionCustomizations.qll

@@ -90,7 +90,7 @@ module LogInjection {
     // TODO: Consider rewriting using flow states.
     ReplaceLineBreaksSanitizer() {
       this.getFunction().(DataFlow::AttrRead).getAttributeName() = "replace" and
-      this.getArg(0).asExpr().(StrConst).getText() in ["\r\n", "\n"]
+      this.getArg(0).asExpr().(StringLiteral).getText() in ["\r\n", "\n"]
     }
   }
 }

python/ql/lib/semmle/python/security/dataflow/PamAuthorizationCustomizations.qll

@@ -20,7 +20,7 @@ module PamAuthorizationCustomizations {
     exists(API::CallNode findLibCall, API::CallNode cdllCall |
       findLibCall =
         API::moduleImport("ctypes").getMember("util").getMember("find_library").getACall() and
-      findLibCall.getParameter(0).getAValueReachingSink().asExpr().(StrConst).getText() = "pam" and
+      findLibCall.getParameter(0).getAValueReachingSink().asExpr().(StringLiteral).getText() = "pam" and
       cdllCall = API::moduleImport("ctypes").getMember("CDLL").getACall() and
       cdllCall.getParameter(0).getAValueReachingSink() = findLibCall
     |

python/ql/lib/semmle/python/security/dataflow/ServerSideRequestForgeryCustomizations.qll

@@ -88,7 +88,9 @@ module ServerSideRequestForgery {
       exists(BinaryExprNode add |
         add.getOp() instanceof Add and
         add.getRight() = this.asCfgNode() and
-        not add.getLeft().getNode().(StrConst).getText().toLowerCase() in ["http://", "https://"]
+        not add.getLeft().getNode().(StringLiteral).getText().toLowerCase() in [
+            "http://", "https://"
+          ]
       )
       or
       // % formatting
@@ -97,7 +99,7 @@ module ServerSideRequestForgery {
         fmt.getRight() = this.asCfgNode() and
         // detecting %-formatting is not super easy, so we simplify it to only handle
         // when there is a **single** substitution going on.
-        not fmt.getLeft().getNode().(StrConst).getText().regexpMatch("^(?i)https?://%s[^%]*$")
+        not fmt.getLeft().getNode().(StringLiteral).getText().regexpMatch("^(?i)https?://%s[^%]*$")
       )
       or
       // arguments to a format call
@@ -106,9 +108,9 @@ module ServerSideRequestForgery {
       |
         call.getMethodName() = "format" and
         (
-          if call.getObject().asExpr().(StrConst).getText().regexpMatch(httpPrefixRe)
+          if call.getObject().asExpr().(StringLiteral).getText().regexpMatch(httpPrefixRe)
           then
-            exists(string text | text = call.getObject().asExpr().(StrConst).getText() |
+            exists(string text | text = call.getObject().asExpr().(StringLiteral).getText() |
               // `http://{}...`
               exists(text.regexpCapture(httpPrefixRe, 1)) and
               this in [call.getArg(any(int i | i >= 1)), call.getArgByName(_)]
@@ -129,7 +131,7 @@ module ServerSideRequestForgery {
       or
       // f-string
       exists(Fstring fstring |
-        if fstring.getValue(0).(StrConst).getText().toLowerCase() in ["http://", "https://"]
+        if fstring.getValue(0).(StringLiteral).getText().toLowerCase() in ["http://", "https://"]
         then fstring.getValue(any(int i | i >= 2)) = this.asExpr()
         else fstring.getValue(any(int i | i >= 1)) = this.asExpr()
       )

python/ql/lib/semmle/python/security/dataflow/TarSlipCustomizations.qll

@@ -39,7 +39,7 @@ module TarSlip {
       this = API::moduleImport("tarfile").getMember("open").getACall() and
       // If argument refers to a string object, then it's a hardcoded path and
       // this tarfile is safe.
-      not this.(DataFlow::CallCfgNode).getArg(0).getALocalSource().asExpr() instanceof StrConst and
+      not this.(DataFlow::CallCfgNode).getArg(0).getALocalSource().asExpr() instanceof StringLiteral and
       // Ignore opens within the tarfile module itself
       not this.getLocation().getFile().getBaseName() = "tarfile.py"
     }
@@ -70,7 +70,7 @@ module TarSlip {
       exists(Expr filterValue |
         filterValue = call.getParameter(4, "filter").getAValueReachingSink().asExpr() and
         (
-          filterValue.(StrConst).getText() = "fully_trusted"
+          filterValue.(StringLiteral).getText() = "fully_trusted"
           or
           filterValue instanceof None
         )

python/ql/lib/semmle/python/security/dataflow/UnsafeShellCommandConstructionCustomizations.qll

@@ -33,7 +33,7 @@ module UnsafeShellCommandConstruction {
 
   /** A sink for shell command constructed from library input vulnerabilities. */
   abstract class Sink extends DataFlow::Node {
-    Sink() { not this.asExpr() instanceof StrConst } // filter out string constants, makes testing easier
+    Sink() { not this.asExpr() instanceof StringLiteral } // filter out string constants, makes testing easier
 
     /** Gets a description of how the string in this sink was constructed. */
     abstract string describe();
@@ -113,7 +113,7 @@ module UnsafeShellCommandConstruction {
 
     ArrayJoin() {
       call.getMethodName() = "join" and
-      unique( | | call.getArg(_)).asExpr().(Str).getText() = " " and
+      unique( | | call.getArg(_)).asExpr().(StringLiteral).getText() = " " and
       isUsedAsShellCommand(call, s) and
       (
         this = call.getArg(0) and

python/ql/lib/semmle/python/security/dataflow/UrlRedirectCustomizations.qll

@@ -118,8 +118,8 @@ module UrlRedirect {
 
     ReplaceBackslashesSanitizer() {
       this.calls(receiver, "replace") and
-      this.getArg(0).asExpr().(StrConst).getText() = "\\" and
-      this.getArg(1).asExpr().(StrConst).getText() in ["/", ""]
+      this.getArg(0).asExpr().(StringLiteral).getText() = "\\" and
+      this.getArg(1).asExpr().(StringLiteral).getText() in ["/", ""]
     }
 
     override predicate sanitizes(FlowState state) { state instanceof MayContainBackslashes }

python/ql/lib/semmle/python/security/internal/SensitiveDataHeuristics.qll

@@ -106,6 +106,25 @@ module HeuristicNames {
       "(?is).*([^\\w$.-]|redact|censor|obfuscate|hash|md5|sha|random|((?<!un)(en))?(crypt|(?<!pass)code)|certain|concert|secretar|accountant|accountab).*"
   }
 
+  /**
+   * Holds if `name` may indicate the presence of sensitive data, and `name` does not indicate that
+   * the data is in fact non-sensitive (for example since it is hashed or encrypted).
+   *
+   * That is, one of the regexps from `maybeSensitiveRegexp` matches `name` (with the given
+   * classification), and none of the regexps from `notSensitiveRegexp` matches `name`.
+   */
+  bindingset[name]
+  predicate nameIndicatesSensitiveData(string name) {
+    exists(string combinedRegexp |
+      // Combine all the maybe-sensitive regexps into one using non-capturing groups and |.
+      combinedRegexp =
+        "(?:" + strictconcat(string r | r = maybeSensitiveRegexp(_) | r, ")|(?:") + ")"
+    |
+      name.regexpMatch(combinedRegexp)
+    ) and
+    not name.regexpMatch(notSensitiveRegexp())
+  }
+
   /**
    * Holds if `name` may indicate the presence of sensitive data, and
    * `name` does not indicate that the data is in fact non-sensitive (for example since
@@ -115,6 +134,10 @@ module HeuristicNames {
    * That is, one of the regexps from `maybeSensitiveRegexp` matches `name` (with the
    * given classification), and none of the regexps from `notSensitiveRegexp` matches
    * `name`.
+   *
+   * When the set of names is large, it's worth using `nameIndicatesSensitiveData/1` as a first
+   * pass, since that combines all the regexps into one, and should be faster. Then call this
+   * predicate to get the classification(s).
    */
   bindingset[name]
   predicate nameIndicatesSensitiveData(string name, SensitiveDataClassification classification) {

python/ql/lib/semmle/python/strings.qll

@@ -1,10 +1,10 @@
 import python
 
-predicate format_string(StrConst e) {
+predicate format_string(StringLiteral e) {
   exists(BinaryExpr b | b.getOp() instanceof Mod and b.getLeft() = e)
 }
 
-predicate mapping_format(StrConst e) {
+predicate mapping_format(StringLiteral e) {
   conversion_specifier(e, _).regexpMatch("%\\([A-Z_a-z0-9]+\\).*")
 }
 
@@ -17,18 +17,18 @@ predicate mapping_format(StrConst e) {
  * TYPE = "[bdiouxXeEfFgGcrs%]"
  */
 
-private string conversion_specifier_string(StrConst e, int number, int position) {
+private string conversion_specifier_string(StringLiteral e, int number, int position) {
   exists(string s, string regex | s = e.getText() |
     regex = "%(\\([^)]*\\))?[#0\\- +]*(\\*|[0-9]*)(\\.(\\*|[0-9]*))?(h|H|l|L)?[badiouxXeEfFgGcrs%]" and
     result = s.regexpFind(regex, number, position)
   )
 }
 
-private string conversion_specifier(StrConst e, int number) {
+private string conversion_specifier(StringLiteral e, int number) {
   result = conversion_specifier_string(e, number, _) and result != "%%"
 }
 
-int illegal_conversion_specifier(StrConst e) {
+int illegal_conversion_specifier(StringLiteral e) {
   format_string(e) and
   "%" = e.getText().charAt(result) and
   // not the start of a conversion specifier or the second % of a %%
@@ -37,7 +37,7 @@ int illegal_conversion_specifier(StrConst e) {
 }
 
 /** Gets the number of format items in a format string */
-int format_items(StrConst e) {
+int format_items(StringLiteral e) {
   result =
     count(int i | | conversion_specifier(e, i)) +
       // a conversion specifier uses an extra item for each *
@@ -47,7 +47,7 @@ int format_items(StrConst e) {
 private string str(Expr e) {
   result = e.(Num).getN()
   or
-  result = "'" + e.(StrConst).getText() + "'"
+  result = "'" + e.(StringLiteral).getText() + "'"
 }
 
 /** Gets a string representation of an expression more suited for embedding in message strings than .toString() */

python/ql/lib/semmle/python/types/ModuleKind.qll

@@ -15,7 +15,7 @@ private predicate is_script(ModuleObject m) {
   (
     m.getModule().getFile().getExtension() != ".py"
     or
-    exists(If i, Name name, StrConst main, Cmpop op |
+    exists(If i, Name name, StringLiteral main, Cmpop op |
       i.getScope() = m.getModule() and
       op instanceof Eq and
       i.getTest().(Compare).compares(name, op, main) and

python/ql/lib/semmle/python/types/Object.qll

@@ -9,7 +9,7 @@ private predicate is_an_object(@py_object obj) {
   /* CFG nodes for numeric literals, all of which have a @py_cobject for the value of that literal */
   obj instanceof ControlFlowNode and
   not obj.(ControlFlowNode).getNode() instanceof IntegerLiteral and
-  not obj.(ControlFlowNode).getNode() instanceof StrConst
+  not obj.(ControlFlowNode).getNode() instanceof StringLiteral
   or
   obj instanceof Builtin
 }

python/ql/lib/semmle/python/xml/XML.qll

@@ -3,305 +3,67 @@
  */
 
 import semmle.files.FileSystem
+private import codeql.xml.Xml
 
-private class TXmlLocatable =
-  @xmldtd or @xmlelement or @xmlattribute or @xmlnamespace or @xmlcomment or @xmlcharacters;
+private module Input implements InputSig<File, Location> {
+  class XmlLocatableBase = @xmllocatable or @xmlnamespaceable;
 
-/** An XML element that has a location. */
-class XmlLocatable extends @xmllocatable, TXmlLocatable {
-  /** Gets the source location for this element. */
-  Location getLocation() { xmllocations(this, result) }
+  predicate xmllocations_(XmlLocatableBase e, Location loc) { xmllocations(e, loc) }
 
-  /**
-   * Holds if this element is at the specified location.
-   * The location spans column `startcolumn` of line `startline` to
-   * column `endcolumn` of line `endline` in file `filepath`.
-   * For more information, see
-   * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/).
-   */
-  predicate hasLocationInfo(
-    string filepath, int startline, int startcolumn, int endline, int endcolumn
+  class XmlParentBase = @xmlparent;
+
+  class XmlNamespaceableBase = @xmlnamespaceable;
+
+  class XmlElementBase = @xmlelement;
+
+  class XmlFileBase = File;
+
+  predicate xmlEncoding_(XmlFileBase f, string enc) { xmlEncoding(f, enc) }
+
+  class XmlDtdBase = @xmldtd;
+
+  predicate xmlDTDs_(XmlDtdBase e, string root, string publicId, string systemId, XmlFileBase file) {
+    xmlDTDs(e, root, publicId, systemId, file)
+  }
+
+  predicate xmlElements_(
+    XmlElementBase e, string name, XmlParentBase parent, int idx, XmlFileBase file
   ) {
-    exists(File f, Location l | l = this.getLocation() |
-      locations_default(l, f, startline, startcolumn, endline, endcolumn) and
-      filepath = f.getAbsolutePath()
-    )
+    xmlElements(e, name, parent, idx, file)
   }
 
-  /** Gets a textual representation of this element. */
-  string toString() { none() } // overridden in subclasses
-}
+  class XmlAttributeBase = @xmlattribute;
 
-/**
- * An `XmlParent` is either an `XmlElement` or an `XmlFile`,
- * both of which can contain other elements.
- */
-class XmlParent extends @xmlparent {
-  XmlParent() {
-    // explicitly restrict `this` to be either an `XmlElement` or an `XmlFile`;
-    // the type `@xmlparent` currently also includes non-XML files
-    this instanceof @xmlelement or xmlEncoding(this, _)
+  predicate xmlAttrs_(
+    XmlAttributeBase e, XmlElementBase elementid, string name, string value, int idx,
+    XmlFileBase file
+  ) {
+    xmlAttrs(e, elementid, name, value, idx, file)
   }
 
-  /**
-   * Gets a printable representation of this XML parent.
-   * (Intended to be overridden in subclasses.)
-   */
-  string getName() { none() } // overridden in subclasses
+  class XmlNamespaceBase = @xmlnamespace;
 
-  /** Gets the file to which this XML parent belongs. */
-  XmlFile getFile() { result = this or xmlElements(this, _, _, _, result) }
-
-  /** Gets the child element at a specified index of this XML parent. */
-  XmlElement getChild(int index) { xmlElements(result, _, this, index, _) }
-
-  /** Gets a child element of this XML parent. */
-  XmlElement getAChild() { xmlElements(result, _, this, _, _) }
-
-  /** Gets a child element of this XML parent with the given `name`. */
-  XmlElement getAChild(string name) { xmlElements(result, _, this, _, _) and result.hasName(name) }
-
-  /** Gets a comment that is a child of this XML parent. */
-  XmlComment getAComment() { xmlComments(result, _, this, _) }
-
-  /** Gets a character sequence that is a child of this XML parent. */
-  XmlCharacters getACharactersSet() { xmlChars(result, _, this, _, _, _) }
-
-  /** Gets the depth in the tree. (Overridden in XmlElement.) */
-  int getDepth() { result = 0 }
-
-  /** Gets the number of child XML elements of this XML parent. */
-  int getNumberOfChildren() { result = count(XmlElement e | xmlElements(e, _, this, _, _)) }
-
-  /** Gets the number of places in the body of this XML parent where text occurs. */
-  int getNumberOfCharacterSets() { result = count(int pos | xmlChars(_, _, this, pos, _, _)) }
-
-  /**
-   * Gets the result of appending all the character sequences of this XML parent from
-   * left to right, separated by a space.
-   */
-  string allCharactersString() {
-    result =
-      concat(string chars, int pos | xmlChars(_, chars, this, pos, _, _) | chars, " " order by pos)
+  predicate xmlNs_(XmlNamespaceBase e, string prefixName, string uri, XmlFileBase file) {
+    xmlNs(e, prefixName, uri, file)
   }
 
-  /** Gets the text value contained in this XML parent. */
-  string getTextValue() { result = this.allCharactersString() }
+  predicate xmlHasNs_(XmlNamespaceableBase e, XmlNamespaceBase ns, XmlFileBase file) {
+    xmlHasNs(e, ns, file)
+  }
 
-  /** Gets a printable representation of this XML parent. */
-  string toString() { result = this.getName() }
-}
+  class XmlCommentBase = @xmlcomment;
 
-/** An XML file. */
-class XmlFile extends XmlParent, File {
-  XmlFile() { xmlEncoding(this, _) }
+  predicate xmlComments_(XmlCommentBase e, string text, XmlParentBase parent, XmlFileBase file) {
+    xmlComments(e, text, parent, file)
+  }
 
-  /** Gets a printable representation of this XML file. */
-  override string toString() { result = this.getName() }
+  class XmlCharactersBase = @xmlcharacters;
 
-  /** Gets the name of this XML file. */
-  override string getName() { result = File.super.getAbsolutePath() }
-
-  /** Gets the encoding of this XML file. */
-  string getEncoding() { xmlEncoding(this, result) }
-
-  /** Gets the XML file itself. */
-  override XmlFile getFile() { result = this }
-
-  /** Gets a top-most element in an XML file. */
-  XmlElement getARootElement() { result = this.getAChild() }
-
-  /** Gets a DTD associated with this XML file. */
-  XmlDtd getADtd() { xmlDTDs(result, _, _, _, this) }
-}
-
-/**
- * An XML document type definition (DTD).
- *
- * Example:
- *
- * ```
- * <!ELEMENT person (firstName, lastName?)>
- * <!ELEMENT firstName (#PCDATA)>
- * <!ELEMENT lastName (#PCDATA)>
- * ```
- */
-class XmlDtd extends XmlLocatable, @xmldtd {
-  /** Gets the name of the root element of this DTD. */
-  string getRoot() { xmlDTDs(this, result, _, _, _) }
-
-  /** Gets the public ID of this DTD. */
-  string getPublicId() { xmlDTDs(this, _, result, _, _) }
-
-  /** Gets the system ID of this DTD. */
-  string getSystemId() { xmlDTDs(this, _, _, result, _) }
-
-  /** Holds if this DTD is public. */
-  predicate isPublic() { not xmlDTDs(this, _, "", _, _) }
-
-  /** Gets the parent of this DTD. */
-  XmlParent getParent() { xmlDTDs(this, _, _, _, result) }
-
-  override string toString() {
-    this.isPublic() and
-    result = this.getRoot() + " PUBLIC '" + this.getPublicId() + "' '" + this.getSystemId() + "'"
-    or
-    not this.isPublic() and
-    result = this.getRoot() + " SYSTEM '" + this.getSystemId() + "'"
+  predicate xmlChars_(
+    XmlCharactersBase e, string text, XmlParentBase parent, int idx, int isCDATA, XmlFileBase file
+  ) {
+    xmlChars(e, text, parent, idx, isCDATA, file)
   }
 }
 
-/**
- * An XML element in an XML file.
- *
- * Example:
- *
- * ```
- * <manifest xmlns:android="http://schemas.android.com/apk/res/android"
- *           package="com.example.exampleapp" android:versionCode="1">
- * </manifest>
- * ```
- */
-class XmlElement extends @xmlelement, XmlParent, XmlLocatable {
-  /** Holds if this XML element has the given `name`. */
-  predicate hasName(string name) { name = this.getName() }
-
-  /** Gets the name of this XML element. */
-  override string getName() { xmlElements(this, result, _, _, _) }
-
-  /** Gets the XML file in which this XML element occurs. */
-  override XmlFile getFile() { xmlElements(this, _, _, _, result) }
-
-  /** Gets the parent of this XML element. */
-  XmlParent getParent() { xmlElements(this, _, result, _, _) }
-
-  /** Gets the index of this XML element among its parent's children. */
-  int getIndex() { xmlElements(this, _, _, result, _) }
-
-  /** Holds if this XML element has a namespace. */
-  predicate hasNamespace() { xmlHasNs(this, _, _) }
-
-  /** Gets the namespace of this XML element, if any. */
-  XmlNamespace getNamespace() { xmlHasNs(this, result, _) }
-
-  /** Gets the index of this XML element among its parent's children. */
-  int getElementPositionIndex() { xmlElements(this, _, _, result, _) }
-
-  /** Gets the depth of this element within the XML file tree structure. */
-  override int getDepth() { result = this.getParent().getDepth() + 1 }
-
-  /** Gets an XML attribute of this XML element. */
-  XmlAttribute getAnAttribute() { result.getElement() = this }
-
-  /** Gets the attribute with the specified `name`, if any. */
-  XmlAttribute getAttribute(string name) { result.getElement() = this and result.getName() = name }
-
-  /** Holds if this XML element has an attribute with the specified `name`. */
-  predicate hasAttribute(string name) { exists(this.getAttribute(name)) }
-
-  /** Gets the value of the attribute with the specified `name`, if any. */
-  string getAttributeValue(string name) { result = this.getAttribute(name).getValue() }
-
-  /** Gets a printable representation of this XML element. */
-  override string toString() { result = this.getName() }
-}
-
-/**
- * An attribute that occurs inside an XML element.
- *
- * Examples:
- *
- * ```
- * package="com.example.exampleapp"
- * android:versionCode="1"
- * ```
- */
-class XmlAttribute extends @xmlattribute, XmlLocatable {
-  /** Gets the name of this attribute. */
-  string getName() { xmlAttrs(this, _, result, _, _, _) }
-
-  /** Gets the XML element to which this attribute belongs. */
-  XmlElement getElement() { xmlAttrs(this, result, _, _, _, _) }
-
-  /** Holds if this attribute has a namespace. */
-  predicate hasNamespace() { xmlHasNs(this, _, _) }
-
-  /** Gets the namespace of this attribute, if any. */
-  XmlNamespace getNamespace() { xmlHasNs(this, result, _) }
-
-  /** Gets the value of this attribute. */
-  string getValue() { xmlAttrs(this, _, _, result, _, _) }
-
-  /** Gets a printable representation of this XML attribute. */
-  override string toString() { result = this.getName() + "=" + this.getValue() }
-}
-
-/**
- * A namespace used in an XML file.
- *
- * Example:
- *
- * ```
- * xmlns:android="http://schemas.android.com/apk/res/android"
- * ```
- */
-class XmlNamespace extends XmlLocatable, @xmlnamespace {
-  /** Gets the prefix of this namespace. */
-  string getPrefix() { xmlNs(this, result, _, _) }
-
-  /** Gets the URI of this namespace. */
-  string getUri() { xmlNs(this, _, result, _) }
-
-  /** Holds if this namespace has no prefix. */
-  predicate isDefault() { this.getPrefix() = "" }
-
-  override string toString() {
-    this.isDefault() and result = this.getUri()
-    or
-    not this.isDefault() and result = this.getPrefix() + ":" + this.getUri()
-  }
-}
-
-/**
- * A comment in an XML file.
- *
- * Example:
- *
- * ```
- * <!-- This is a comment. -->
- * ```
- */
-class XmlComment extends @xmlcomment, XmlLocatable {
-  /** Gets the text content of this XML comment. */
-  string getText() { xmlComments(this, result, _, _) }
-
-  /** Gets the parent of this XML comment. */
-  XmlParent getParent() { xmlComments(this, _, result, _) }
-
-  /** Gets a printable representation of this XML comment. */
-  override string toString() { result = this.getText() }
-}
-
-/**
- * A sequence of characters that occurs between opening and
- * closing tags of an XML element, excluding other elements.
- *
- * Example:
- *
- * ```
- * <content>This is a sequence of characters.</content>
- * ```
- */
-class XmlCharacters extends @xmlcharacters, XmlLocatable {
-  /** Gets the content of this character sequence. */
-  string getCharacters() { xmlChars(this, result, _, _, _, _) }
-
-  /** Gets the parent of this character sequence. */
-  XmlParent getParent() { xmlChars(this, _, result, _, _, _) }
-
-  /** Holds if this character sequence is CDATA. */
-  predicate isCDATA() { xmlChars(this, _, _, _, 1, _) }
-
-  /** Gets a printable representation of this XML character sequence. */
-  override string toString() { result = this.getCharacters() }
-}
+import Make<File, Location, Input>