Merge branch 'main' into python/enable-summaries-from-models

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowDispatch.qll

@@ -251,6 +251,9 @@ abstract class LibraryCallable extends string {
   /** Gets a call to this library callable. */
   abstract CallCfgNode getACall();
 
+  /** Same as `getACall` but without referring to the call graph or API graph. */
+  CallCfgNode getACallSimple() { none() }
+
   /** Gets a data-flow node, where this library callable is used as a call-back. */
   abstract ArgumentNode getACallback();
 }

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl.qll

@@ -2021,7 +2021,8 @@ module Impl<FullStateConfigSig Config> {
       FlowCheckNode() {
         castNode(this.asNode()) or
         clearsContentCached(this.asNode(), _) or
-        expectsContentCached(this.asNode(), _)
+        expectsContentCached(this.asNode(), _) or
+        neverSkipInPathGraph(this.asNode())
       }
     }
 

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowPrivate.qll

@@ -486,6 +486,14 @@ class DataFlowType extends TDataFlowType {
 
 /** A node that performs a type cast. */
 class CastNode extends Node {
+  CastNode() { none() }
+}
+
+/**
+ * Holds if `n` should never be skipped over in the `PathGraph` and in path
+ * explanations.
+ */
+predicate neverSkipInPathGraph(Node n) {
   // We include read- and store steps here to force them to be
   // shown in path explanations.
   // This hack is necessary, because we have included some of these
@@ -494,7 +502,7 @@ class CastNode extends Node {
   // We should revert this once, we can remove this steps from the
   // default taint steps; this should be possible once we have
   // implemented flow summaries and recursive content.
-  CastNode() { readStep(_, _, this) or storeStep(_, _, this) }
+  readStep(_, _, n) or storeStep(_, _, n)
 }
 
 /**

python/ql/lib/semmle/python/dataflow/new/internal/SummaryTypeTracker.qll

@@ -0,0 +1,391 @@
+/**
+ * Provides the implementation of type tracking steps through flow summaries.
+ * To use this, you must implement the `Input` signature. You can then use the predicates in the `Output`
+ * signature to implement the predicates of the same names inside `TypeTrackerSpecific.qll`.
+ */
+
+/** The classes and predicates needed to generate type-tracking steps from summaries. */
+signature module Input {
+  // Dataflow nodes
+  class Node;
+
+  // Content
+  class TypeTrackerContent;
+
+  class TypeTrackerContentFilter;
+
+  // Relating content and filters
+  /**
+   * Gets a content filter to use for a `WithoutContent[content]` step, (data is not allowed to be stored in `content`)
+   * or has no result if
+   * the step should be treated as ordinary flow.
+   *
+   * `WithoutContent` is often used to perform strong updates on individual collection elements, but for
+   * type-tracking this is rarely beneficial and quite expensive. However, `WithoutContent` can be quite useful
+   * for restricting the type of an object, and in these cases we translate it to a filter.
+   */
+  TypeTrackerContentFilter getFilterFromWithoutContentStep(TypeTrackerContent content);
+
+  /**
+   * Gets a content filter to use for a `WithContent[content]` step, (data must be stored in `content`)
+   * or has no result if
+   * the step cannot be handled by type-tracking.
+   *
+   * `WithContent` is often used to perform strong updates on individual collection elements (or rather
+   * to preserve those that didn't get updated). But for type-tracking this is rarely beneficial and quite expensive.
+   * However, `WithContent` can be quite useful for restricting the type of an object, and in these cases we translate it to a filter.
+   */
+  TypeTrackerContentFilter getFilterFromWithContentStep(TypeTrackerContent content);
+
+  // Summaries and their stacks
+  class SummaryComponent;
+
+  class SummaryComponentStack {
+    SummaryComponent head();
+  }
+
+  /** Gets a singleton stack containing `component`. */
+  SummaryComponentStack singleton(SummaryComponent component);
+
+  /**
+   * Gets the stack obtained by pushing `head` onto `tail`.
+   */
+  SummaryComponentStack push(SummaryComponent head, SummaryComponentStack tail);
+
+  /** Gets a singleton stack representing a return. */
+  SummaryComponent return();
+
+  // Relating content to summaries
+  /** Gets a summary component for content `c`. */
+  SummaryComponent content(TypeTrackerContent contents);
+
+  /** Gets a summary component where data is not allowed to be stored in `contents`. */
+  SummaryComponent withoutContent(TypeTrackerContent contents);
+
+  /** Gets a summary component where data must be stored in `contents`. */
+  SummaryComponent withContent(TypeTrackerContent contents);
+
+  // Callables
+  class SummarizedCallable {
+    predicate propagatesFlow(
+      SummaryComponentStack input, SummaryComponentStack output, boolean preservesValue
+    );
+  }
+
+  // Relating nodes to summaries
+  /** Gets a dataflow node respresenting the argument of `call` indicated by `arg`. */
+  Node argumentOf(Node call, SummaryComponent arg);
+
+  /** Gets a dataflow node respresenting the parameter of `callable` indicated by `param`. */
+  Node parameterOf(Node callable, SummaryComponent param);
+
+  /** Gets a dataflow node respresenting the return of `callable` indicated by `return`. */
+  Node returnOf(Node callable, SummaryComponent return);
+
+  // Relating callables to nodes
+  /** Gets a dataflow node respresenting a call to `callable`. */
+  Node callTo(SummarizedCallable callable);
+}
+
+/**
+ * The predicates provided by a summary type tracker.
+ * These are meant to be used in `TypeTrackerSpecific.qll`
+ * inside the predicates of the same names.
+ */
+signature module Output<Input I> {
+  /**
+   *  Holds if there is a level step from `nodeFrom` to `nodeTo`, which does not depend on the call graph.
+   */
+  predicate levelStepNoCall(I::Node nodeFrom, I::Node nodeTo);
+
+  /**
+   * Holds if `nodeTo` is the result of accessing the `content` content of `nodeFrom`.
+   */
+  predicate basicLoadStep(I::Node nodeFrom, I::Node nodeTo, I::TypeTrackerContent content);
+
+  /**
+   * Holds if `nodeFrom` is being written to the `content` content of the object in `nodeTo`.
+   */
+  predicate basicStoreStep(I::Node nodeFrom, I::Node nodeTo, I::TypeTrackerContent content);
+
+  /**
+   * Holds if the `loadContent` of `nodeFrom` is stored in the `storeContent` of `nodeTo`.
+   */
+  predicate basicLoadStoreStep(
+    I::Node nodeFrom, I::Node nodeTo, I::TypeTrackerContent loadContent,
+    I::TypeTrackerContent storeContent
+  );
+
+  /**
+   * Holds if type-tracking should step from `nodeFrom` to `nodeTo` but block flow of contents matched by `filter` through here.
+   */
+  predicate basicWithoutContentStep(
+    I::Node nodeFrom, I::Node nodeTo, I::TypeTrackerContentFilter filter
+  );
+
+  /**
+   * Holds if type-tracking should step from `nodeFrom` to `nodeTo` if inside a content matched by `filter`.
+   */
+  predicate basicWithContentStep(
+    I::Node nodeFrom, I::Node nodeTo, I::TypeTrackerContentFilter filter
+  );
+}
+
+/**
+ * Implementation of the summary type tracker, that is type tracking through flow summaries.
+ */
+module SummaryFlow<Input I> implements Output<I> {
+  pragma[nomagic]
+  private predicate isNonLocal(I::SummaryComponent component) {
+    component = I::content(_)
+    or
+    component = I::withContent(_)
+  }
+
+  pragma[nomagic]
+  private predicate hasLoadSummary(
+    I::SummarizedCallable callable, I::TypeTrackerContent contents, I::SummaryComponentStack input,
+    I::SummaryComponentStack output
+  ) {
+    callable.propagatesFlow(I::push(I::content(contents), input), output, true) and
+    not isNonLocal(input.head()) and
+    not isNonLocal(output.head())
+  }
+
+  pragma[nomagic]
+  private predicate hasStoreSummary(
+    I::SummarizedCallable callable, I::TypeTrackerContent contents, I::SummaryComponentStack input,
+    I::SummaryComponentStack output
+  ) {
+    not isNonLocal(input.head()) and
+    not isNonLocal(output.head()) and
+    (
+      callable.propagatesFlow(input, I::push(I::content(contents), output), true)
+      or
+      // Allow the input to start with an arbitrary WithoutContent[X].
+      // Since type-tracking only tracks one content deep, and we're about to store into another content,
+      // we're already preventing the input from being in a content.
+      callable
+          .propagatesFlow(I::push(I::withoutContent(_), input),
+            I::push(I::content(contents), output), true)
+    )
+  }
+
+  pragma[nomagic]
+  private predicate hasLoadStoreSummary(
+    I::SummarizedCallable callable, I::TypeTrackerContent loadContents,
+    I::TypeTrackerContent storeContents, I::SummaryComponentStack input,
+    I::SummaryComponentStack output
+  ) {
+    callable
+        .propagatesFlow(I::push(I::content(loadContents), input),
+          I::push(I::content(storeContents), output), true) and
+    not isNonLocal(input.head()) and
+    not isNonLocal(output.head())
+  }
+
+  pragma[nomagic]
+  private predicate hasWithoutContentSummary(
+    I::SummarizedCallable callable, I::TypeTrackerContentFilter filter,
+    I::SummaryComponentStack input, I::SummaryComponentStack output
+  ) {
+    exists(I::TypeTrackerContent content |
+      callable.propagatesFlow(I::push(I::withoutContent(content), input), output, true) and
+      filter = I::getFilterFromWithoutContentStep(content) and
+      not isNonLocal(input.head()) and
+      not isNonLocal(output.head()) and
+      input != output
+    )
+  }
+
+  pragma[nomagic]
+  private predicate hasWithContentSummary(
+    I::SummarizedCallable callable, I::TypeTrackerContentFilter filter,
+    I::SummaryComponentStack input, I::SummaryComponentStack output
+  ) {
+    exists(I::TypeTrackerContent content |
+      callable.propagatesFlow(I::push(I::withContent(content), input), output, true) and
+      filter = I::getFilterFromWithContentStep(content) and
+      not isNonLocal(input.head()) and
+      not isNonLocal(output.head()) and
+      input != output
+    )
+  }
+
+  private predicate componentLevelStep(I::SummaryComponent component) {
+    exists(I::TypeTrackerContent content |
+      component = I::withoutContent(content) and
+      not exists(I::getFilterFromWithoutContentStep(content))
+    )
+  }
+
+  /**
+   * Gets a data flow `I::Node` corresponding an argument or return value of `call`,
+   * as specified by `component`.
+   */
+  bindingset[call, component]
+  private I::Node evaluateSummaryComponentLocal(I::Node call, I::SummaryComponent component) {
+    result = I::argumentOf(call, component)
+    or
+    component = I::return() and
+    result = call
+  }
+
+  /**
+   * Holds if `callable` is relevant for type-tracking and we therefore want `stack` to
+   * be evaluated locally at its call sites.
+   */
+  pragma[nomagic]
+  private predicate dependsOnSummaryComponentStack(
+    I::SummarizedCallable callable, I::SummaryComponentStack stack
+  ) {
+    exists(I::callTo(callable)) and
+    (
+      callable.propagatesFlow(stack, _, true)
+      or
+      callable.propagatesFlow(_, stack, true)
+      or
+      // include store summaries as they may skip an initial step at the input
+      hasStoreSummary(callable, _, stack, _)
+    )
+    or
+    dependsOnSummaryComponentStackCons(callable, _, stack)
+  }
+
+  pragma[nomagic]
+  private predicate dependsOnSummaryComponentStackCons(
+    I::SummarizedCallable callable, I::SummaryComponent head, I::SummaryComponentStack tail
+  ) {
+    dependsOnSummaryComponentStack(callable, I::push(head, tail))
+  }
+
+  pragma[nomagic]
+  private predicate dependsOnSummaryComponentStackConsLocal(
+    I::SummarizedCallable callable, I::SummaryComponent head, I::SummaryComponentStack tail
+  ) {
+    dependsOnSummaryComponentStackCons(callable, head, tail) and
+    not isNonLocal(head)
+  }
+
+  pragma[nomagic]
+  private predicate dependsOnSummaryComponentStackLeaf(
+    I::SummarizedCallable callable, I::SummaryComponent leaf
+  ) {
+    dependsOnSummaryComponentStack(callable, I::singleton(leaf))
+  }
+
+  /**
+   * Gets a data flow I::Node corresponding to the local input or output of `call`
+   * identified by `stack`, if possible.
+   */
+  pragma[nomagic]
+  private I::Node evaluateSummaryComponentStackLocal(
+    I::SummarizedCallable callable, I::Node call, I::SummaryComponentStack stack
+  ) {
+    exists(I::SummaryComponent component |
+      dependsOnSummaryComponentStackLeaf(callable, component) and
+      stack = I::singleton(component) and
+      call = I::callTo(callable) and
+      result = evaluateSummaryComponentLocal(call, component)
+    )
+    or
+    exists(I::Node prev, I::SummaryComponent head, I::SummaryComponentStack tail |
+      prev = evaluateSummaryComponentStackLocal(callable, call, tail) and
+      dependsOnSummaryComponentStackConsLocal(callable, pragma[only_bind_into](head),
+        pragma[only_bind_out](tail)) and
+      stack = I::push(pragma[only_bind_out](head), pragma[only_bind_out](tail))
+    |
+      result = I::parameterOf(prev, head)
+      or
+      result = I::returnOf(prev, head)
+      or
+      componentLevelStep(head) and
+      result = prev
+    )
+  }
+
+  // Implement Output
+  predicate levelStepNoCall(I::Node nodeFrom, I::Node nodeTo) {
+    exists(
+      I::SummarizedCallable callable, I::Node call, I::SummaryComponentStack input,
+      I::SummaryComponentStack output
+    |
+      callable.propagatesFlow(input, output, true) and
+      call = I::callTo(callable) and
+      nodeFrom = evaluateSummaryComponentStackLocal(callable, call, input) and
+      nodeTo = evaluateSummaryComponentStackLocal(callable, call, output)
+    )
+  }
+
+  predicate basicLoadStep(I::Node nodeFrom, I::Node nodeTo, I::TypeTrackerContent content) {
+    exists(
+      I::SummarizedCallable callable, I::Node call, I::SummaryComponentStack input,
+      I::SummaryComponentStack output
+    |
+      hasLoadSummary(callable, content, pragma[only_bind_into](input),
+        pragma[only_bind_into](output)) and
+      call = I::callTo(callable) and
+      nodeFrom = evaluateSummaryComponentStackLocal(callable, call, input) and
+      nodeTo = evaluateSummaryComponentStackLocal(callable, call, output)
+    )
+  }
+
+  predicate basicStoreStep(I::Node nodeFrom, I::Node nodeTo, I::TypeTrackerContent content) {
+    exists(
+      I::SummarizedCallable callable, I::Node call, I::SummaryComponentStack input,
+      I::SummaryComponentStack output
+    |
+      hasStoreSummary(callable, content, pragma[only_bind_into](input),
+        pragma[only_bind_into](output)) and
+      call = I::callTo(callable) and
+      nodeFrom = evaluateSummaryComponentStackLocal(callable, call, input) and
+      nodeTo = evaluateSummaryComponentStackLocal(callable, call, output)
+    )
+  }
+
+  predicate basicLoadStoreStep(
+    I::Node nodeFrom, I::Node nodeTo, I::TypeTrackerContent loadContent,
+    I::TypeTrackerContent storeContent
+  ) {
+    exists(
+      I::SummarizedCallable callable, I::Node call, I::SummaryComponentStack input,
+      I::SummaryComponentStack output
+    |
+      hasLoadStoreSummary(callable, loadContent, storeContent, pragma[only_bind_into](input),
+        pragma[only_bind_into](output)) and
+      call = I::callTo(callable) and
+      nodeFrom = evaluateSummaryComponentStackLocal(callable, call, input) and
+      nodeTo = evaluateSummaryComponentStackLocal(callable, call, output)
+    )
+  }
+
+  predicate basicWithoutContentStep(
+    I::Node nodeFrom, I::Node nodeTo, I::TypeTrackerContentFilter filter
+  ) {
+    exists(
+      I::SummarizedCallable callable, I::Node call, I::SummaryComponentStack input,
+      I::SummaryComponentStack output
+    |
+      hasWithoutContentSummary(callable, filter, pragma[only_bind_into](input),
+        pragma[only_bind_into](output)) and
+      call = I::callTo(callable) and
+      nodeFrom = evaluateSummaryComponentStackLocal(callable, call, input) and
+      nodeTo = evaluateSummaryComponentStackLocal(callable, call, output)
+    )
+  }
+
+  predicate basicWithContentStep(
+    I::Node nodeFrom, I::Node nodeTo, I::TypeTrackerContentFilter filter
+  ) {
+    exists(
+      I::SummarizedCallable callable, I::Node call, I::SummaryComponentStack input,
+      I::SummaryComponentStack output
+    |
+      hasWithContentSummary(callable, filter, pragma[only_bind_into](input),
+        pragma[only_bind_into](output)) and
+      call = I::callTo(callable) and
+      nodeFrom = evaluateSummaryComponentStackLocal(callable, call, input) and
+      nodeTo = evaluateSummaryComponentStackLocal(callable, call, output)
+    )
+  }
+}

python/ql/lib/semmle/python/dataflow/new/internal/TypeTrackerSpecific.qll

@@ -61,7 +61,9 @@ predicate capturedJumpStep(Node nodeFrom, Node nodeTo) {
 predicate levelStepCall(Node nodeFrom, Node nodeTo) { none() }
 
 /** Holds if there is a level step from `nodeFrom` to `nodeTo`, which does not depend on the call graph. */
-predicate levelStepNoCall(Node nodeFrom, Node nodeTo) { none() }
+predicate levelStepNoCall(Node nodeFrom, Node nodeTo) {
+  TypeTrackerSummaryFlow::levelStepNoCall(nodeFrom, nodeTo)
+}
 
 /**
  * Gets the name of a possible piece of content. For Python, this is currently only attribute names,
@@ -108,6 +110,12 @@ predicate basicStoreStep(Node nodeFrom, Node nodeTo, string content) {
     nodeFrom = a.getValue() and
     nodeTo = a.getObject()
   )
+  or
+  exists(DataFlowPublic::ContentSet contents |
+    contents.(DataFlowPublic::AttributeContent).getAttribute() = content
+  |
+    TypeTrackerSummaryFlow::basicStoreStep(nodeFrom, nodeTo, contents)
+  )
 }
 
 /**
@@ -119,13 +127,24 @@ predicate basicLoadStep(Node nodeFrom, Node nodeTo, string content) {
     nodeFrom = a.getObject() and
     nodeTo = a
   )
+  or
+  exists(DataFlowPublic::ContentSet contents |
+    contents.(DataFlowPublic::AttributeContent).getAttribute() = content
+  |
+    TypeTrackerSummaryFlow::basicLoadStep(nodeFrom, nodeTo, contents)
+  )
 }
 
 /**
  * Holds if the `loadContent` of `nodeFrom` is stored in the `storeContent` of `nodeTo`.
  */
 predicate basicLoadStoreStep(Node nodeFrom, Node nodeTo, string loadContent, string storeContent) {
-  none()
+  exists(DataFlowPublic::ContentSet loadContents, DataFlowPublic::ContentSet storeContents |
+    loadContents.(DataFlowPublic::AttributeContent).getAttribute() = loadContent and
+    storeContents.(DataFlowPublic::AttributeContent).getAttribute() = storeContent
+  |
+    TypeTrackerSummaryFlow::basicLoadStoreStep(nodeFrom, nodeTo, loadContents, storeContents)
+  )
 }
 
 /**
@@ -144,3 +163,93 @@ predicate basicWithContentStep(Node nodeFrom, Node nodeTo, ContentFilter filter)
 class Boolean extends boolean {
   Boolean() { this = true or this = false }
 }
+
+private import SummaryTypeTracker as SummaryTypeTracker
+private import semmle.python.dataflow.new.FlowSummary as FlowSummary
+private import semmle.python.dataflow.new.internal.DataFlowDispatch as DataFlowDispatch
+
+pragma[noinline]
+private predicate argumentPositionMatch(
+  DataFlowPublic::CallCfgNode call, DataFlowPublic::Node arg,
+  DataFlowDispatch::ParameterPosition ppos
+) {
+  exists(DataFlowDispatch::ArgumentPosition apos |
+    DataFlowDispatch::parameterMatch(ppos, apos) and
+    DataFlowDispatch::normalCallArg(call.getNode(), arg, apos)
+  )
+}
+
+private module SummaryTypeTrackerInput implements SummaryTypeTracker::Input {
+  // Dataflow nodes
+  class Node = DataFlowPublic::Node;
+
+  // Content
+  class TypeTrackerContent = DataFlowPublic::ContentSet;
+
+  class TypeTrackerContentFilter = ContentFilter;
+
+  TypeTrackerContentFilter getFilterFromWithoutContentStep(TypeTrackerContent content) { none() }
+
+  TypeTrackerContentFilter getFilterFromWithContentStep(TypeTrackerContent content) { none() }
+
+  // Callables
+  class SummarizedCallable = FlowSummary::SummarizedCallable;
+
+  // Summaries and their stacks
+  class SummaryComponent = FlowSummary::SummaryComponent;
+
+  class SummaryComponentStack = FlowSummary::SummaryComponentStack;
+
+  predicate singleton = FlowSummary::SummaryComponentStack::singleton/1;
+
+  predicate push = FlowSummary::SummaryComponentStack::push/2;
+
+  // Relating content to summaries
+  predicate content = FlowSummary::SummaryComponent::content/1;
+
+  SummaryComponent withoutContent(TypeTrackerContent contents) { none() }
+
+  SummaryComponent withContent(TypeTrackerContent contents) { none() }
+
+  predicate return = FlowSummary::SummaryComponent::return/0;
+
+  // Relating nodes to summaries
+  Node argumentOf(Node call, SummaryComponent arg) {
+    exists(DataFlowDispatch::ParameterPosition pos |
+      arg = FlowSummary::SummaryComponent::argument(pos) and
+      argumentPositionMatch(call, result, pos)
+    )
+  }
+
+  Node parameterOf(Node callable, SummaryComponent param) {
+    exists(
+      DataFlowDispatch::ArgumentPosition apos, DataFlowDispatch::ParameterPosition ppos, Parameter p
+    |
+      param = FlowSummary::SummaryComponent::parameter(apos) and
+      DataFlowDispatch::parameterMatch(ppos, apos) and
+      // pick the SsaNode rather than the CfgNode
+      result.asVar().getDefinition().(ParameterDefinition).getParameter() = p and
+      (
+        exists(int i | ppos.isPositional(i) |
+          p = callable.getALocalSource().asExpr().(CallableExpr).getInnerScope().getArg(i)
+        )
+        or
+        exists(string name | ppos.isKeyword(name) |
+          p = callable.getALocalSource().asExpr().(CallableExpr).getInnerScope().getArgByName(name)
+        )
+      )
+    )
+  }
+
+  Node returnOf(Node callable, SummaryComponent return) {
+    return = FlowSummary::SummaryComponent::return() and
+    // `result` should be the return value of a callable expression (lambda or function) referenced by `callable`
+    result.asCfgNode() =
+      callable.getALocalSource().asExpr().(CallableExpr).getInnerScope().getAReturnValueFlowNode()
+  }
+
+  // Relating callables to nodes
+  Node callTo(SummarizedCallable callable) { result = callable.getACallSimple() }
+}
+
+private module TypeTrackerSummaryFlow = SummaryTypeTracker::SummaryFlow<SummaryTypeTrackerInput>;

python/ql/lib/semmle/python/dataflow/old/TaintTracking.qll

@@ -664,6 +664,14 @@ module DataFlow {
   }
 }
 
+deprecated private class DataFlowType extends TaintKind {
+  // this only exists to avoid an empty recursion error in the type checker
+  DataFlowType() {
+    this = "Data flow" and
+    1 = 2
+  }
+}
+
 pragma[noinline]
 private predicate dict_construct(ControlFlowNode itemnode, ControlFlowNode dictnode) {
   dictnode.(DictNode).getAValue() = itemnode

python/ql/lib/semmle/python/frameworks/data/internal/ApiGraphModels.qll

@@ -662,6 +662,17 @@ module ModelOutput {
 
   import Cached
   import Specific::ModelOutputSpecific
+  private import codeql.mad.ModelValidation as SharedModelVal
+
+  private module KindValConfig implements SharedModelVal::KindValidationConfigSig {
+    predicate summaryKind(string kind) { summaryModel(_, _, _, _, kind) }
+
+    predicate sinkKind(string kind) { sinkModel(_, _, kind) }
+
+    predicate sourceKind(string kind) { sourceModel(_, _, kind) }
+  }
+
+  private module KindVal = SharedModelVal::KindValidation<KindValConfig>;
 
   /**
    * Gets an error message relating to an invalid CSV row in a model.
@@ -707,5 +718,8 @@ module ModelOutput {
       not isValidNoArgumentTokenInIdentifyingAccessPath(token.getName()) and
       result = "Invalid token '" + token + "' is missing its arguments, in access path: " + path
     )
+    or
+    // Check for invalid model kinds
+    result = KindVal::getInvalidModelKind()
   }
 }

python/ql/lib/semmle/python/security/Exceptions.qll

@@ -1,103 +0,0 @@
-/**
- * Provides classes and predicates for tracking exceptions and information
- * associated with exceptions.
- */
-
-import python
-import semmle.python.dataflow.TaintTracking
-import semmle.python.security.strings.Basic
-
-deprecated private Value traceback_function(string name) {
-  result = Module::named("traceback").attr(name)
-}
-
-/**
- * This represents information relating to an exception, for instance the
- * message, arguments or parts of the exception traceback.
- */
-deprecated class ExceptionInfo extends StringKind {
-  ExceptionInfo() { this = "exception.info" }
-
-  override string repr() { result = "exception info" }
-}
-
-/**
- * A class representing sources of information about
- * execution state exposed in tracebacks and the like.
- */
-abstract deprecated class ErrorInfoSource extends TaintSource { }
-
-/**
- * This kind represents exceptions themselves.
- */
-deprecated class ExceptionKind extends TaintKind {
-  ExceptionKind() { this = "exception.kind" }
-
-  override string repr() { result = "exception" }
-
-  override TaintKind getTaintOfAttribute(string name) {
-    name = "args" and result instanceof ExceptionInfoSequence
-    or
-    name = "message" and result instanceof ExceptionInfo
-  }
-}
-
-/**
- * A source of exception objects, either explicitly created, or captured by an
- * `except` statement.
- */
-deprecated class ExceptionSource extends ErrorInfoSource {
-  ExceptionSource() {
-    exists(ClassValue cls |
-      cls.getASuperType() = ClassValue::baseException() and
-      this.(ControlFlowNode).pointsTo().getClass() = cls
-    )
-    or
-    this = any(ExceptStmt s).getName().getAFlowNode()
-  }
-
-  override string toString() { result = "exception.source" }
-
-  override predicate isSourceOf(TaintKind kind) { kind instanceof ExceptionKind }
-}
-
-/**
- * Represents a sequence of pieces of information relating to an exception,
- * for instance the contents of the `args` attribute, or the stack trace.
- */
-deprecated class ExceptionInfoSequence extends SequenceKind {
-  ExceptionInfoSequence() { this.getItem() instanceof ExceptionInfo }
-}
-
-/**
- * Represents calls to functions in the `traceback` module that return
- * sequences of exception information.
- */
-deprecated class CallToTracebackFunction extends ErrorInfoSource {
-  CallToTracebackFunction() {
-    exists(string name |
-      name in [
-          "extract_tb", "extract_stack", "format_list", "format_exception_only", "format_exception",
-          "format_tb", "format_stack"
-        ]
-    |
-      this = traceback_function(name).getACall()
-    )
-  }
-
-  override string toString() { result = "exception.info.sequence.source" }
-
-  override predicate isSourceOf(TaintKind kind) { kind instanceof ExceptionInfoSequence }
-}
-
-/**
- * Represents calls to functions in the `traceback` module that return a single
- * string of information about an exception.
- */
-deprecated class FormattedTracebackSource extends ErrorInfoSource {
-  FormattedTracebackSource() { this = traceback_function("format_exc").getACall() }
-
-  override string toString() { result = "exception.info.source" }
-
-  override predicate isSourceOf(TaintKind kind) { kind instanceof ExceptionInfo }
-}

python/ql/lib/semmle/python/security/SQL.qll

@@ -1,4 +1,4 @@
 import python
 import semmle.python.dataflow.TaintTracking
 
-abstract class SqlInjectionSink extends TaintSink { }
+abstract deprecated class SqlInjectionSink extends TaintSink { }

python/ql/lib/semmle/python/security/injection/Command.qll

@@ -1,263 +0,0 @@
-/**
- * Provides class and predicates to track external data that
- * may represent malicious OS commands.
- *
- * This module is intended to be imported into a taint-tracking query
- * to extend `TaintKind` and `TaintSink`.
- */
-
-import python
-import semmle.python.dataflow.TaintTracking
-import semmle.python.security.strings.Untrusted
-
-/** Abstract taint sink that is potentially vulnerable to malicious shell commands. */
-abstract deprecated class CommandSink extends TaintSink { }
-
-deprecated private ModuleObject osOrPopenModule() { result.getName() = ["os", "popen2"] }
-
-deprecated private Object makeOsCall() {
-  exists(string name | result = ModuleObject::named("subprocess").attr(name) |
-    name = ["Popen", "call", "check_call", "check_output", "run"]
-  )
-}
-
-/**Special case for first element in sequence. */
-deprecated class FirstElementKind extends TaintKind {
-  FirstElementKind() { this = "sequence[" + any(ExternalStringKind key) + "][0]" }
-
-  override string repr() { result = "first item in sequence of " + this.getItem().repr() }
-
-  /** Gets the taint kind for item in this sequence. */
-  ExternalStringKind getItem() { this = "sequence[" + result + "][0]" }
-}
-
-deprecated class FirstElementFlow extends DataFlowExtension::DataFlowNode {
-  FirstElementFlow() { this = any(SequenceNode s).getElement(0) }
-
-  override ControlFlowNode getASuccessorNode(TaintKind fromkind, TaintKind tokind) {
-    result.(SequenceNode).getElement(0) = this and tokind.(FirstElementKind).getItem() = fromkind
-  }
-}
-
-/**
- * A taint sink that is potentially vulnerable to malicious shell commands.
- * The `vuln` in `subprocess.call(shell=vuln)` and similar calls.
- */
-deprecated class ShellCommand extends CommandSink {
-  override string toString() { result = "shell command" }
-
-  ShellCommand() {
-    exists(CallNode call, Object istrue |
-      call.getFunction().refersTo(makeOsCall()) and
-      call.getAnArg() = this and
-      call.getArgByName("shell").refersTo(istrue) and
-      istrue.booleanValue() = true
-    )
-    or
-    exists(CallNode call, string name |
-      call.getAnArg() = this and
-      call.getFunction().refersTo(osOrPopenModule().attr(name))
-    |
-      name = ["system", "popen"] or
-      name.matches("popen_")
-    )
-    or
-    exists(CallNode call |
-      call.getAnArg() = this and
-      call.getFunction().refersTo(ModuleObject::named("commands"))
-    )
-  }
-
-  override predicate sinks(TaintKind kind) {
-    /* Tainted string command */
-    kind instanceof ExternalStringKind
-    or
-    /* List (or tuple) containing a tainted string command */
-    kind instanceof ExternalStringSequenceKind
-  }
-}
-
-/**
- * A taint sink that is potentially vulnerable to malicious shell commands.
- * The `vuln` in `subprocess.call(vuln, ...)` and similar calls.
- */
-deprecated class OsCommandFirstArgument extends CommandSink {
-  override string toString() { result = "OS command first argument" }
-
-  OsCommandFirstArgument() {
-    not this instanceof ShellCommand and
-    exists(CallNode call |
-      call.getFunction().refersTo(makeOsCall()) and
-      call.getArg(0) = this
-    )
-  }
-
-  override predicate sinks(TaintKind kind) {
-    /* Tainted string command */
-    kind instanceof ExternalStringKind
-    or
-    /* List (or tuple) whose first element is tainted */
-    kind instanceof FirstElementKind
-  }
-}
-
-// -------------------------------------------------------------------------- //
-// Modeling of the 'invoke' package and 'fabric' package (v 2.x)
-//
-// Since fabric build so closely upon invoke, we model them together to avoid
-// duplication
-// -------------------------------------------------------------------------- //
-/**
- * A taint sink that is potentially vulnerable to malicious shell commands.
- * The `vuln` in `invoke.run(vuln, ...)` and similar calls.
- */
-deprecated class InvokeRun extends CommandSink {
-  InvokeRun() {
-    this = Value::named("invoke.run").(FunctionValue).getArgumentForCall(_, 0)
-    or
-    this = Value::named("invoke.sudo").(FunctionValue).getArgumentForCall(_, 0)
-  }
-
-  override string toString() { result = "InvokeRun" }
-
-  override predicate sinks(TaintKind kind) { kind instanceof ExternalStringKind }
-}
-
-/**
- * Internal TaintKind to track the invoke.Context instance passed to functions
- * marked with @invoke.task
- */
-deprecated private class InvokeContextArg extends TaintKind {
-  InvokeContextArg() { this = "InvokeContextArg" }
-}
-
-/** Internal TaintSource to track the context passed to functions marked with @invoke.task */
-deprecated private class InvokeContextArgSource extends TaintSource {
-  InvokeContextArgSource() {
-    exists(Function f, Expr decorator |
-      count(f.getADecorator()) = 1 and
-      (
-        decorator = f.getADecorator() and not decorator instanceof Call
-        or
-        decorator = f.getADecorator().(Call).getFunc()
-      ) and
-      (
-        decorator.pointsTo(Value::named("invoke.task"))
-        or
-        decorator.pointsTo(Value::named("fabric.task"))
-      )
-    |
-      this.(ControlFlowNode).getNode() = f.getArg(0)
-    )
-  }
-
-  override predicate isSourceOf(TaintKind kind) { kind instanceof InvokeContextArg }
-}
-
-/**
- * A taint sink that is potentially vulnerable to malicious shell commands.
- * The `vuln` in `invoke.Context().run(vuln, ...)` and similar calls.
- */
-deprecated class InvokeContextRun extends CommandSink {
-  InvokeContextRun() {
-    exists(CallNode call |
-      any(InvokeContextArg k).taints(call.getFunction().(AttrNode).getObject("run"))
-      or
-      call = Value::named("invoke.Context").(ClassValue).lookup("run").getACall()
-      or
-      // fabric.connection.Connection is a subtype of invoke.context.Context
-      // since fabric.Connection.run has a decorator, it doesn't work with FunctionValue :|
-      // and `Value::named("fabric.Connection").(ClassValue).lookup("run").getACall()` returned no results,
-      // so here is the hacky solution that works :\
-      call.getFunction().(AttrNode).getObject("run").pointsTo().getClass() =
-        Value::named("fabric.Connection")
-    |
-      this = call.getArg(0)
-      or
-      this = call.getArgByName("command")
-    )
-  }
-
-  override string toString() { result = "InvokeContextRun" }
-
-  override predicate sinks(TaintKind kind) { kind instanceof ExternalStringKind }
-}
-
-/**
- * A taint sink that is potentially vulnerable to malicious shell commands.
- * The `vuln` in `fabric.Group().run(vuln, ...)` and similar calls.
- */
-deprecated class FabricGroupRun extends CommandSink {
-  FabricGroupRun() {
-    exists(ClassValue cls |
-      cls.getASuperType() = Value::named("fabric.Group") and
-      this = cls.lookup("run").(FunctionValue).getArgumentForCall(_, 1)
-    )
-  }
-
-  override string toString() { result = "FabricGroupRun" }
-
-  override predicate sinks(TaintKind kind) { kind instanceof ExternalStringKind }
-}
-
-// -------------------------------------------------------------------------- //
-// Modeling of the 'invoke' package and 'fabric' package (v 1.x)
-// -------------------------------------------------------------------------- //
-deprecated class FabricV1Commands extends CommandSink {
-  FabricV1Commands() {
-    // since `run` and `sudo` are decorated, we can't use FunctionValue's :(
-    exists(CallNode call |
-      call = Value::named("fabric.api.local").getACall()
-      or
-      call = Value::named("fabric.api.run").getACall()
-      or
-      call = Value::named("fabric.api.sudo").getACall()
-    |
-      this = call.getArg(0)
-      or
-      this = call.getArgByName("command")
-    )
-  }
-
-  override string toString() { result = "FabricV1Commands" }
-
-  override predicate sinks(TaintKind kind) { kind instanceof ExternalStringKind }
-}
-
-/**
- * An extension that propagates taint from the arguments of `fabric.api.execute(func, arg0, arg1, ...)`
- * to the parameters of `func`, since this will call `func(arg0, arg1, ...)`.
- */
-deprecated class FabricExecuteExtension extends DataFlowExtension::DataFlowNode {
-  CallNode call;
-
-  FabricExecuteExtension() {
-    call = Value::named("fabric.api.execute").getACall() and
-    (
-      this = call.getArg(any(int i | i > 0))
-      or
-      this = call.getArgByName(any(string s | not s = "task"))
-    )
-  }
-
-  override ControlFlowNode getASuccessorNode(TaintKind fromkind, TaintKind tokind) {
-    tokind = fromkind and
-    exists(CallableValue func |
-      (
-        call.getArg(0).pointsTo(func)
-        or
-        call.getArgByName("task").pointsTo(func)
-      ) and
-      exists(int i |
-        // execute(func, arg0, arg1) => func(arg0, arg1)
-        this = call.getArg(i) and
-        result = func.getParameter(i - 1)
-      )
-      or
-      exists(string name |
-        this = call.getArgByName(name) and
-        result = func.getParameterByName(name)
-      )
-    )
-  }
-}

python/ql/lib/semmle/python/security/injection/Sql.qll

@@ -1,83 +0,0 @@
-/**
- * Provides class and predicates to track external data that
- * may represent malicious SQL queries or parts of queries.
- *
- * This module is intended to be imported into a taint-tracking query
- * to extend `TaintKind` and `TaintSink`.
- */
-
-import python
-import semmle.python.dataflow.TaintTracking
-import semmle.python.security.strings.Untrusted
-import semmle.python.security.SQL
-
-deprecated private StringObject first_part(ControlFlowNode command) {
-  command.(BinaryExprNode).getOp() instanceof Add and
-  command.(BinaryExprNode).getLeft().refersTo(result)
-  or
-  exists(CallNode call, SequenceObject seq | call = command |
-    call = theStrType().lookupAttribute("join") and
-    call.getArg(0).refersTo(seq) and
-    seq.getInferredElement(0) = result
-  )
-  or
-  command.(BinaryExprNode).getOp() instanceof Mod and
-  command.getNode().(StrConst).getLiteralObject() = result
-}
-
-/** Holds if `command` appears to be a SQL command string of which `inject` is a part. */
-deprecated predicate probable_sql_command(ControlFlowNode command, ControlFlowNode inject) {
-  exists(string prefix |
-    inject = command.getAChild*() and
-    first_part(command).getText().regexpMatch(" *" + prefix + ".*")
-  |
-    prefix = "CREATE" or prefix = "SELECT"
-  )
-}
-
-/**
- * A taint kind representing a DB cursor.
- * This will be overridden to provide specific kinds of DB cursor.
- */
-abstract deprecated class DbCursor extends TaintKind {
-  bindingset[this]
-  DbCursor() { any() }
-
-  string getExecuteMethodName() { result = "execute" }
-}
-
-/**
- * A part of a string that appears to be a SQL command and is thus
- * vulnerable to malicious input.
- */
-deprecated class SimpleSqlStringInjection extends SqlInjectionSink {
-  override string toString() { result = "simple SQL string injection" }
-
-  SimpleSqlStringInjection() { probable_sql_command(_, this) }
-
-  override predicate sinks(TaintKind kind) { kind instanceof ExternalStringKind }
-}
-
-/**
- * A taint source representing sources of DB connections.
- * This will be overridden to provide specific kinds of DB connection sources.
- */
-abstract deprecated class DbConnectionSource extends TaintSource { }
-
-/**
- * A taint sink that is vulnerable to malicious SQL queries.
- * The `vuln` in `db.connection.execute(vuln)` and similar.
- */
-deprecated class DbConnectionExecuteArgument extends SqlInjectionSink {
-  override string toString() { result = "db.connection.execute" }
-
-  DbConnectionExecuteArgument() {
-    exists(CallNode call, DbCursor cursor, string name |
-      cursor.taints(call.getFunction().(AttrNode).getObject(name)) and
-      cursor.getExecuteMethodName() = name and
-      call.getArg(0) = this
-    )
-  }
-
-  override predicate sinks(TaintKind kind) { kind instanceof ExternalStringKind }
-}

python/ql/lib/semmle/python/web/ClientHttpRequest.qll

@@ -1,2 +0,0 @@
-import semmle.python.web.client.StdLib
-import semmle.python.web.client.Requests

python/ql/lib/semmle/python/web/HttpRedirect.qll

@@ -1,7 +0,0 @@
-import python
-import semmle.python.security.strings.Basic
-import semmle.python.web.django.Redirect
-import semmle.python.web.flask.Redirect
-import semmle.python.web.tornado.Redirect
-import semmle.python.web.pyramid.Redirect
-import semmle.python.web.bottle.Redirect

python/ql/lib/semmle/python/web/HttpResponse.qll

@@ -1,10 +0,0 @@
-import semmle.python.web.django.Response
-import semmle.python.web.flask.Response
-import semmle.python.web.pyramid.Response
-import semmle.python.web.tornado.Response
-import semmle.python.web.twisted.Response
-import semmle.python.web.bottle.Response
-import semmle.python.web.turbogears.Response
-import semmle.python.web.falcon.Response
-import semmle.python.web.cherrypy.Response
-import semmle.python.web.stdlib.Response

python/ql/lib/semmle/python/web/bottle/Redirect.qll

@@ -1,28 +0,0 @@
-/**
- * Provides class representing the `bottle.redirect` function.
- * This module is intended to be imported into a taint-tracking query
- * to extend `TaintSink`.
- */
-
-import python
-import semmle.python.dataflow.TaintTracking
-import semmle.python.security.strings.Basic
-import semmle.python.web.bottle.General
-
-deprecated FunctionValue bottle_redirect() { result = theBottleModule().attr("redirect") }
-
-/**
- * An argument to the `bottle.redirect` function.
- */
-deprecated class BottleRedirect extends TaintSink {
-  override string toString() { result = "bottle.redirect" }
-
-  BottleRedirect() {
-    exists(CallNode call |
-      bottle_redirect().getACall() = call and
-      this = call.getAnArg()
-    )
-  }
-
-  override predicate sinks(TaintKind kind) { kind instanceof StringKind }
-}

python/ql/lib/semmle/python/web/bottle/Response.qll

@@ -1,52 +0,0 @@
-import python
-import semmle.python.dataflow.TaintTracking
-import semmle.python.security.strings.Untrusted
-import semmle.python.web.Http
-import semmle.python.web.bottle.General
-
-/**
- * A bottle.Response object
- * This isn't really a "taint", but we use the value tracking machinery to
- * track the flow of response objects.
- */
-deprecated class BottleResponse extends TaintKind {
-  BottleResponse() { this = "bottle.response" }
-}
-
-deprecated private Value theBottleResponseObject() { result = theBottleModule().attr("response") }
-
-deprecated class BottleResponseBodyAssignment extends HttpResponseTaintSink {
-  BottleResponseBodyAssignment() {
-    exists(DefinitionNode lhs |
-      lhs.getValue() = this and
-      lhs.(AttrNode).getObject("body").pointsTo(theBottleResponseObject())
-    )
-  }
-
-  override predicate sinks(TaintKind kind) { kind instanceof StringKind }
-}
-
-deprecated class BottleHandlerFunctionResult extends HttpResponseTaintSink {
-  BottleHandlerFunctionResult() {
-    exists(BottleRoute route, Return ret |
-      ret.getScope() = route.getFunction() and
-      ret.getValue().getAFlowNode() = this
-    )
-  }
-
-  override predicate sinks(TaintKind kind) { kind instanceof StringKind }
-
-  override string toString() { result = "bottle handler function result" }
-}
-
-deprecated class BottleCookieSet extends CookieSet, CallNode {
-  BottleCookieSet() {
-    any(BottleResponse r).taints(this.getFunction().(AttrNode).getObject("set_cookie"))
-  }
-
-  override string toString() { result = CallNode.super.toString() }
-
-  override ControlFlowNode getKey() { result = this.getArg(0) }
-
-  override ControlFlowNode getValue() { result = this.getArg(1) }
-}

python/ql/lib/semmle/python/web/cherrypy/Response.qll

@@ -1,18 +0,0 @@
-import python
-import semmle.python.dataflow.TaintTracking
-import semmle.python.security.strings.Untrusted
-import semmle.python.web.Http
-import semmle.python.web.cherrypy.General
-
-deprecated class CherryPyExposedFunctionResult extends HttpResponseTaintSink {
-  CherryPyExposedFunctionResult() {
-    exists(Return ret |
-      ret.getScope() instanceof CherryPyExposedFunction and
-      ret.getValue().getAFlowNode() = this
-    )
-  }
-
-  override predicate sinks(TaintKind kind) { kind instanceof StringKind }
-
-  override string toString() { result = "cherrypy handler function result" }
-}

python/ql/lib/semmle/python/web/client/Requests.qll

@@ -1,22 +0,0 @@
-/**
- * Modeling outgoing HTTP requests using the `requests` package
- * https://pypi.org/project/requests/
- */
-
-import python
-private import semmle.python.web.Http
-
-deprecated class RequestsHttpRequest extends Client::HttpRequest, CallNode {
-  CallableValue func;
-  string method;
-
-  RequestsHttpRequest() {
-    method = httpVerbLower() and
-    func = Module::named("requests").attr(method) and
-    this = func.getACall()
-  }
-
-  override ControlFlowNode getAUrlPart() { result = func.getNamedArgumentForCall(this, "url") }
-
-  override string getMethodUpper() { result = method.toUpperCase() }
-}

python/ql/lib/semmle/python/web/client/StdLib.qll

@@ -1,55 +0,0 @@
-import python
-private import semmle.python.web.Http
-
-deprecated ClassValue httpConnectionClass() {
-  // Python 2
-  result = Value::named("httplib.HTTPConnection")
-  or
-  result = Value::named("httplib.HTTPSConnection")
-  or
-  // Python 3
-  result = Value::named("http.client.HTTPConnection")
-  or
-  result = Value::named("http.client.HTTPSConnection")
-  or
-  // six
-  result = Value::named("six.moves.http_client.HTTPConnection")
-  or
-  result = Value::named("six.moves.http_client.HTTPSConnection")
-}
-
-deprecated class HttpConnectionHttpRequest extends Client::HttpRequest, CallNode {
-  CallNode constructor_call;
-  CallableValue func;
-
-  HttpConnectionHttpRequest() {
-    exists(ClassValue cls, AttrNode call_origin, Value constructor_call_value |
-      cls = httpConnectionClass() and
-      func = cls.lookup("request") and
-      this = func.getACall() and
-      // since you can do `r = conn.request; r('GET', path)`, we need to find the origin
-      this.getFunction().pointsTo(_, _, call_origin) and
-      // Since HTTPSConnection is a subtype of HTTPConnection, up until this point, `cls` could be either class,
-      // because `HTTPSConnection.request == HTTPConnection.request`. To avoid generating 2 results, we filter
-      // on the actual class used as the constructor
-      call_origin.getObject().pointsTo(_, constructor_call_value, constructor_call) and
-      cls = constructor_call_value.getClass() and
-      constructor_call = cls.getACall()
-    )
-  }
-
-  override ControlFlowNode getAUrlPart() {
-    result = func.getNamedArgumentForCall(this, "url")
-    or
-    result = constructor_call.getArg(0)
-    or
-    result = constructor_call.getArgByName("host")
-  }
-
-  override string getMethodUpper() {
-    exists(string method |
-      result = method.toUpperCase() and
-      func.getNamedArgumentForCall(this, "method").pointsTo(Value::forString(method))
-    )
-  }
-}

python/ql/lib/semmle/python/web/django/Db.qll

@@ -1,50 +0,0 @@
-import python
-import semmle.python.security.injection.Sql
-
-/**
- * A taint kind representing a django cursor object.
- */
-deprecated class DjangoDbCursor extends DbCursor {
-  DjangoDbCursor() { this = "django.db.connection.cursor" }
-}
-
-deprecated private Value theDjangoConnectionObject() {
-  result = Value::named("django.db.connection")
-}
-
-/**
- * A kind of taint source representing sources of django cursor objects.
- */
-deprecated class DjangoDbCursorSource extends DbConnectionSource {
-  DjangoDbCursorSource() {
-    exists(AttrNode cursor |
-      this.(CallNode).getFunction() = cursor and
-      cursor.getObject("cursor").pointsTo(theDjangoConnectionObject())
-    )
-  }
-
-  override string toString() { result = "django.db.connection.cursor" }
-
-  override predicate isSourceOf(TaintKind kind) { kind instanceof DjangoDbCursor }
-}
-
-deprecated ClassValue theDjangoRawSqlClass() {
-  result = Value::named("django.db.models.expressions.RawSQL")
-}
-
-/**
- * A sink of taint on calls to `django.db.models.expressions.RawSQL`. This
- * allows arbitrary SQL statements to be executed, which is a security risk.
- */
-deprecated class DjangoRawSqlSink extends SqlInjectionSink {
-  DjangoRawSqlSink() {
-    exists(CallNode call |
-      call = theDjangoRawSqlClass().getACall() and
-      this = call.getArg(0)
-    )
-  }
-
-  override predicate sinks(TaintKind kind) { kind instanceof ExternalStringKind }
-
-  override string toString() { result = "django.db.models.expressions.RawSQL(sink,...)" }
-}

python/ql/lib/semmle/python/web/django/Model.qll

@@ -1,69 +0,0 @@
-import python
-import semmle.python.dataflow.TaintTracking
-import semmle.python.security.strings.Basic
-import semmle.python.web.Http
-import semmle.python.security.injection.Sql
-
-/** A django model class */
-deprecated class DjangoModel extends ClassValue {
-  DjangoModel() { Value::named("django.db.models.Model") = this.getASuperType() }
-}
-
-/** A "taint" for django database tables */
-deprecated class DjangoDbTableObjects extends TaintKind {
-  DjangoDbTableObjects() { this = "django.db.models.Model.objects" }
-
-  override TaintKind getTaintOfMethodResult(string name) {
-    result = this and
-    name in [
-        "filter", "exclude", "none", "all", "union", "intersection", "difference", "select_related",
-        "prefetch_related", "extra", "defer", "only", "annotate", "using", "select_for_update",
-        "raw", "order_by", "reverse", "distinct", "values", "values_list", "dates", "datetimes"
-      ]
-  }
-}
-
-/** Django model objects, which are sources of django database table "taint" */
-deprecated class DjangoModelObjects extends TaintSource {
-  DjangoModelObjects() {
-    this.(AttrNode).isLoad() and this.(AttrNode).getObject("objects").pointsTo(any(DjangoModel m))
-  }
-
-  override predicate isSourceOf(TaintKind kind) { kind instanceof DjangoDbTableObjects }
-
-  override string toString() { result = "django.db.models.Model.objects" }
-}
-
-/**
- * A call to the `raw` method on a django model. This allows a raw SQL query
- * to be sent to the database, which is a security risk.
- */
-deprecated class DjangoModelRawCall extends SqlInjectionSink {
-  DjangoModelRawCall() {
-    exists(CallNode raw_call, ControlFlowNode queryset | this = raw_call.getArg(0) |
-      raw_call.getFunction().(AttrNode).getObject("raw") = queryset and
-      any(DjangoDbTableObjects objs).taints(queryset)
-    )
-  }
-
-  override predicate sinks(TaintKind kind) { kind instanceof ExternalStringKind }
-
-  override string toString() { result = "django.models.QuerySet.raw(sink,...)" }
-}
-
-/**
- * A call to the `extra` method on a django model. This allows a raw SQL query
- * to be sent to the database, which is a security risk.
- */
-deprecated class DjangoModelExtraCall extends SqlInjectionSink {
-  DjangoModelExtraCall() {
-    exists(CallNode extra_call, ControlFlowNode queryset | this = extra_call.getArg(0) |
-      extra_call.getFunction().(AttrNode).getObject("extra") = queryset and
-      any(DjangoDbTableObjects objs).taints(queryset)
-    )
-  }
-
-  override predicate sinks(TaintKind kind) { kind instanceof ExternalStringKind }
-
-  override string toString() { result = "django.models.QuerySet.extra(sink,...)" }
-}

python/ql/lib/semmle/python/web/django/Redirect.qll

@@ -1,37 +0,0 @@
-/**
- * Provides class representing the `django.redirect` function.
- * This module is intended to be imported into a taint-tracking query
- * to extend `TaintSink`.
- */
-
-import python
-import semmle.python.dataflow.TaintTracking
-import semmle.python.security.strings.Basic
-private import semmle.python.web.django.Shared
-private import semmle.python.web.Http
-
-/**
- * The URL argument for a call to the `django.shortcuts.redirect` function.
- */
-deprecated class DjangoShortcutsRedirectSink extends HttpRedirectTaintSink {
-  override string toString() { result = "DjangoShortcutsRedirectSink" }
-
-  DjangoShortcutsRedirectSink() {
-    this = Value::named("django.shortcuts.redirect").(FunctionValue).getArgumentForCall(_, 0)
-  }
-}
-
-/**
- * The URL argument when instantiating a Django Redirect Response.
- */
-deprecated class DjangoRedirectResponseSink extends HttpRedirectTaintSink {
-  DjangoRedirectResponseSink() {
-    exists(CallNode call | call = any(DjangoRedirectResponseClass cls).getACall() |
-      this = call.getArg(0)
-      or
-      this = call.getArgByName("redirect_to")
-    )
-  }
-
-  override string toString() { result = "DjangoRedirectResponseSink" }
-}

python/ql/lib/semmle/python/web/django/Response.qll

@@ -1,79 +0,0 @@
-import python
-import semmle.python.dataflow.TaintTracking
-import semmle.python.security.strings.Basic
-private import semmle.python.web.django.Shared
-private import semmle.python.web.Http
-
-/** INTERNAL class used for tracking a django response object. */
-deprecated private class DjangoResponseKind extends TaintKind {
-  DjangoResponseKind() { this = "django.response.HttpResponse" }
-}
-
-/** INTERNAL taint-source used for tracking a django response object. */
-deprecated private class DjangoResponseSource extends TaintSource {
-  DjangoResponseSource() { exists(DjangoContentResponseClass cls | cls.getACall() = this) }
-
-  override predicate isSourceOf(TaintKind kind) { kind instanceof DjangoResponseKind }
-
-  override string toString() { result = "django.http.response.HttpResponse" }
-}
-
-/** A write to a django response, which is vulnerable to external data (xss) */
-deprecated class DjangoResponseWrite extends HttpResponseTaintSink {
-  DjangoResponseWrite() {
-    exists(AttrNode meth, CallNode call |
-      call.getFunction() = meth and
-      any(DjangoResponseKind response).taints(meth.getObject("write")) and
-      this = call.getArg(0)
-    )
-  }
-
-  override predicate sinks(TaintKind kind) { kind instanceof StringKind }
-
-  override string toString() { result = "django.Response.write(...)" }
-}
-
-/**
- * An argument to initialization of a django response.
- */
-deprecated class DjangoResponseContent extends HttpResponseTaintSink {
-  DjangoContentResponseClass cls;
-  CallNode call;
-
-  DjangoResponseContent() {
-    call = cls.getACall() and
-    this = cls.getContentArg(call)
-  }
-
-  override predicate sinks(TaintKind kind) { kind instanceof StringKind }
-
-  override string toString() { result = "django.Response(...)" }
-}
-
-/**
- * An argument to initialization of a django response, which is vulnerable to external data (XSS).
- */
-deprecated class DjangoResponseContentXSSVulnerable extends DjangoResponseContent {
-  override DjangoXSSVulnerableResponseClass cls;
-
-  DjangoResponseContentXSSVulnerable() {
-    not exists(cls.getContentTypeArg(call))
-    or
-    exists(StringValue s |
-      cls.getContentTypeArg(call).pointsTo(s) and
-      s.getText().matches("text/html%")
-    )
-  }
-}
-
-deprecated class DjangoCookieSet extends CookieSet, CallNode {
-  DjangoCookieSet() {
-    any(DjangoResponseKind r).taints(this.getFunction().(AttrNode).getObject("set_cookie"))
-  }
-
-  override string toString() { result = CallNode.super.toString() }
-
-  override ControlFlowNode getKey() { result = this.getArg(0) }
-
-  override ControlFlowNode getValue() { result = this.getArg(1) }
-}

python/ql/lib/semmle/python/web/django/Sanitizers.qll

@@ -1,6 +0,0 @@
-import python
-/*
- * Sanitizers
- * No django sanitizers implemented yet.
- */
-

python/ql/lib/semmle/python/web/django/Shared.qll

@@ -1,72 +0,0 @@
-import python
-
-/** A class that is a Django Redirect Response (subclass of `django.http.HttpResponseRedirectBase`). */
-deprecated class DjangoRedirectResponseClass extends ClassValue {
-  DjangoRedirectResponseClass() {
-    exists(ClassValue redirect_base |
-      // version 1.x
-      redirect_base = Value::named("django.http.response.HttpResponseRedirectBase")
-      or
-      // version 2.x and 3.x
-      redirect_base = Value::named("django.http.HttpResponseRedirectBase")
-    |
-      this.getASuperType() = redirect_base
-    )
-  }
-}
-
-/**
- * A class that is a Django Response, which can contain content.
- * A subclass of `django.http.HttpResponse` that is not a `DjangoRedirectResponseClass`.
- */
-deprecated class DjangoContentResponseClass extends ClassValue {
-  ClassValue base;
-
-  DjangoContentResponseClass() {
-    (
-      // version 1.x
-      base = Value::named("django.http.response.HttpResponse")
-      or
-      // version 2.x and 3.x
-      // https://docs.djangoproject.com/en/2.2/ref/request-response/#httpresponse-objects
-      base = Value::named("django.http.HttpResponse")
-    ) and
-    this.getASuperType() = base
-  }
-
-  // The reason these two methods are defined in this class (and not in the Sink
-  // definition that uses this class), is that if we were to add support for
-  // `django.http.response.HttpResponseNotAllowed` it would make much more sense to add
-  // the custom logic in this class (or subclass), than to handle all of it in the sink
-  // definition.
-  /** Gets the `content` argument of a `call` to the constructor */
-  ControlFlowNode getContentArg(CallNode call) { none() }
-
-  /** Gets the `content_type` argument of a `call` to the constructor */
-  ControlFlowNode getContentTypeArg(CallNode call) { none() }
-}
-
-/** A class that is a Django Response, and is vulnerable to XSS. */
-deprecated class DjangoXSSVulnerableResponseClass extends DjangoContentResponseClass {
-  DjangoXSSVulnerableResponseClass() {
-    // We want to avoid FPs on subclasses that are not exposed to XSS, for example `JsonResponse`.
-    // The easiest way is to disregard any subclass that has a special `__init__` method.
-    // It's not guaranteed to remove all FPs, or not to generate FNs, but compared to our
-    // previous implementation that would treat 0-th argument to _any_ subclass as a sink,
-    // this gets us much closer to reality.
-    this.lookup("__init__") = base.lookup("__init__") and
-    not this instanceof DjangoRedirectResponseClass
-  }
-
-  override ControlFlowNode getContentArg(CallNode call) {
-    result = call.getArg(0)
-    or
-    result = call.getArgByName("content")
-  }
-
-  override ControlFlowNode getContentTypeArg(CallNode call) {
-    result = call.getArg(1)
-    or
-    result = call.getArgByName("content_type")
-  }
-}

python/ql/lib/semmle/python/web/falcon/Response.qll

@@ -1,28 +0,0 @@
-import python
-import semmle.python.dataflow.TaintTracking
-import semmle.python.web.Http
-import semmle.python.web.falcon.General
-
-/** https://falcon.readthedocs.io/en/stable/api/request_and_response.html */
-deprecated class FalconResponse extends TaintKind {
-  FalconResponse() { this = "falcon.response" }
-}
-
-/** Only used internally to track the response parameter */
-deprecated private class FalconResponseParameter extends TaintSource {
-  FalconResponseParameter() {
-    exists(FalconHandlerFunction f | f.getResponse() = this.(ControlFlowNode).getNode())
-  }
-
-  override predicate isSourceOf(TaintKind k) { k instanceof FalconResponse }
-}
-
-deprecated class FalconResponseBodySink extends HttpResponseTaintSink {
-  FalconResponseBodySink() {
-    exists(AttrNode attr | any(FalconResponse f).taints(attr.getObject("body")) |
-      attr.(DefinitionNode).getValue() = this
-    )
-  }
-
-  override predicate sinks(TaintKind kind) { kind instanceof StringKind }
-}

python/ql/lib/semmle/python/web/flask/Redirect.qll

@@ -1,26 +0,0 @@
-/**
- * Provides class representing the `flask.redirect` function.
- * This module is intended to be imported into a taint-tracking query
- * to extend `TaintSink`.
- */
-
-import python
-import semmle.python.dataflow.TaintTracking
-import semmle.python.security.strings.Basic
-import semmle.python.web.flask.General
-
-deprecated FunctionValue flask_redirect() { result = Value::named("flask.redirect") }
-
-/**
- * Represents an argument to the `flask.redirect` function.
- */
-deprecated class FlaskRedirect extends HttpRedirectTaintSink {
-  override string toString() { result = "flask.redirect" }
-
-  FlaskRedirect() {
-    exists(CallNode call |
-      flask_redirect().getACall() = call and
-      this = call.getAnArg()
-    )
-  }
-}

python/ql/lib/semmle/python/web/pyramid/Redirect.qll

@@ -1,33 +0,0 @@
-/**
- * Provides class representing the `pyramid.redirect` function.
- * This module is intended to be imported into a taint-tracking query
- * to extend `TaintSink`.
- */
-
-import python
-import semmle.python.dataflow.TaintTracking
-import semmle.python.security.strings.Basic
-import semmle.python.web.Http
-
-deprecated private ClassValue redirectClass() {
-  exists(ModuleValue ex | ex.getName() = "pyramid.httpexceptions" |
-    ex.attr("HTTPFound") = result
-    or
-    ex.attr("HTTPTemporaryRedirect") = result
-  )
-}
-
-/**
- * Represents an argument to the `tornado.redirect` function.
- */
-deprecated class PyramidRedirect extends HttpRedirectTaintSink {
-  override string toString() { result = "pyramid.redirect" }
-
-  PyramidRedirect() {
-    exists(CallNode call | call.getFunction().pointsTo(redirectClass()) |
-      call.getArg(0) = this
-      or
-      call.getArgByName("location") = this
-    )
-  }
-}

python/ql/lib/semmle/python/web/pyramid/Response.qll

@@ -1,37 +0,0 @@
-import python
-import semmle.python.dataflow.TaintTracking
-import semmle.python.security.strings.Basic
-import semmle.python.web.Http
-private import semmle.python.web.pyramid.View
-
-/**
- * A pyramid response, which is vulnerable to any sort of
- * http response malice.
- */
-deprecated class PyramidRoutedResponse extends HttpResponseTaintSink {
-  PyramidRoutedResponse() {
-    exists(PythonFunctionValue view |
-      is_pyramid_view_function(view.getScope()) and
-      this = view.getAReturnedNode()
-    )
-  }
-
-  override predicate sinks(TaintKind kind) { kind instanceof StringKind }
-
-  override string toString() { result = "pyramid.routed.response" }
-}
-
-deprecated class PyramidCookieSet extends CookieSet, CallNode {
-  PyramidCookieSet() {
-    exists(ControlFlowNode f |
-      f = this.getFunction().(AttrNode).getObject("set_cookie") and
-      f.pointsTo().getClass() = Value::named("pyramid.response.Response")
-    )
-  }
-
-  override string toString() { result = CallNode.super.toString() }
-
-  override ControlFlowNode getKey() { result = this.getArg(0) }
-
-  override ControlFlowNode getValue() { result = this.getArg(1) }
-}

python/ql/lib/semmle/python/web/stdlib/Response.qll

@@ -1,43 +0,0 @@
-/**
- * Provides the sinks for HTTP servers defined with standard library (stdlib).
- */
-
-import python
-import semmle.python.dataflow.TaintTracking
-import semmle.python.web.Http
-
-deprecated private predicate is_wfile(AttrNode wfile) {
-  exists(ClassValue cls |
-    // Python 2
-    cls.getABaseType+() = Value::named("BaseHTTPServer.BaseHTTPRequestHandler")
-    or
-    // Python 3
-    cls.getABaseType+() = Value::named("http.server.BaseHTTPRequestHandler")
-  |
-    wfile.getObject("wfile").pointsTo().getClass() = cls
-  )
-}
-
-/** Sink for `h.wfile.write` where `h` is an instance of BaseHttpRequestHandler. */
-deprecated class StdLibWFileWriteSink extends HttpResponseTaintSink {
-  StdLibWFileWriteSink() {
-    exists(CallNode call |
-      is_wfile(call.getFunction().(AttrNode).getObject("write")) and
-      call.getArg(0) = this
-    )
-  }
-
-  override predicate sinks(TaintKind kind) { kind instanceof ExternalStringKind }
-}
-
-/** Sink for `h.wfile.writelines` where `h` is an instance of BaseHttpRequestHandler. */
-deprecated class StdLibWFileWritelinesSink extends HttpResponseTaintSink {
-  StdLibWFileWritelinesSink() {
-    exists(CallNode call |
-      is_wfile(call.getFunction().(AttrNode).getObject("writelines")) and
-      call.getArg(0) = this
-    )
-  }
-
-  override predicate sinks(TaintKind kind) { kind instanceof ExternalStringSequenceKind }
-}

python/ql/lib/semmle/python/web/tornado/Redirect.qll

@@ -1,28 +0,0 @@
-/**
- * Provides class representing the `tornado.redirect` function.
- * This module is intended to be imported into a taint-tracking query
- * to extend `TaintSink`.
- */
-
-import python
-import semmle.python.dataflow.TaintTracking
-import semmle.python.security.strings.Basic
-import semmle.python.web.Http
-import Tornado
-
-/**
- * Represents an argument to the `tornado.redirect` function.
- */
-deprecated class TornadoHttpRequestHandlerRedirect extends HttpRedirectTaintSink {
-  override string toString() { result = "tornado.HttpRequestHandler.redirect" }
-
-  TornadoHttpRequestHandlerRedirect() {
-    exists(CallNode call, ControlFlowNode node |
-      node = call.getFunction().(AttrNode).getObject("redirect") and
-      isTornadoRequestHandlerInstance(node) and
-      this = call.getArg(0)
-    )
-  }
-
-  override predicate sinks(TaintKind kind) { kind instanceof StringKind }
-}

python/ql/lib/semmle/python/web/tornado/Response.qll

@@ -1,47 +0,0 @@
-import python
-import semmle.python.dataflow.TaintTracking
-import semmle.python.security.strings.Basic
-private import semmle.python.web.Http
-import Tornado
-
-deprecated class TornadoConnection extends TaintKind {
-  TornadoConnection() { this = "tornado.http.connection" }
-}
-
-deprecated class TornadoConnectionSource extends TaintSource {
-  TornadoConnectionSource() {
-    isTornadoRequestHandlerInstance(this.(AttrNode).getObject("connection"))
-  }
-
-  override string toString() { result = "Tornado http connection source" }
-
-  override predicate isSourceOf(TaintKind kind) { kind instanceof TornadoConnection }
-}
-
-deprecated class TornadoConnectionWrite extends HttpResponseTaintSink {
-  override string toString() { result = "tornado.connection.write" }
-
-  TornadoConnectionWrite() {
-    exists(CallNode call, ControlFlowNode conn |
-      conn = call.getFunction().(AttrNode).getObject("write") and
-      this = call.getAnArg() and
-      exists(TornadoConnection tc | tc.taints(conn))
-    )
-  }
-
-  override predicate sinks(TaintKind kind) { kind instanceof StringKind }
-}
-
-deprecated class TornadoHttpRequestHandlerWrite extends HttpResponseTaintSink {
-  override string toString() { result = "tornado.HttpRequestHandler.write" }
-
-  TornadoHttpRequestHandlerWrite() {
-    exists(CallNode call, ControlFlowNode node |
-      node = call.getFunction().(AttrNode).getObject("write") and
-      this = call.getAnArg() and
-      isTornadoRequestHandlerInstance(node)
-    )
-  }
-
-  override predicate sinks(TaintKind kind) { kind instanceof StringKind }
-}

python/ql/lib/semmle/python/web/turbogears/Response.qll

@@ -1,31 +0,0 @@
-import python
-import semmle.python.dataflow.TaintTracking
-import semmle.python.security.strings.Basic
-import semmle.python.web.Http
-import TurboGears
-
-deprecated class ControllerMethodReturnValue extends HttpResponseTaintSink {
-  override string toString() { result = "TurboGears ControllerMethodReturnValue" }
-
-  ControllerMethodReturnValue() {
-    exists(TurboGearsControllerMethod m |
-      m.getAReturnValueFlowNode() = this and
-      not m.isTemplated()
-    )
-  }
-
-  override predicate sinks(TaintKind kind) { kind instanceof StringKind }
-}
-
-deprecated class ControllerMethodTemplatedReturnValue extends HttpResponseTaintSink {
-  override string toString() { result = "TurboGears ControllerMethodTemplatedReturnValue" }
-
-  ControllerMethodTemplatedReturnValue() {
-    exists(TurboGearsControllerMethod m |
-      m.getAReturnValueFlowNode() = this and
-      m.isTemplated()
-    )
-  }
-
-  override predicate sinks(TaintKind kind) { kind instanceof ExternalStringDictKind }
-}

python/ql/lib/semmle/python/web/twisted/Response.qll

@@ -1,45 +0,0 @@
-import python
-import semmle.python.dataflow.TaintTracking
-import semmle.python.web.Http
-import semmle.python.security.strings.Basic
-import Twisted
-import Request
-
-deprecated class TwistedResponse extends HttpResponseTaintSink {
-  TwistedResponse() {
-    exists(PythonFunctionValue func, string name |
-      isKnownRequestHandlerMethodName(name) and
-      name = func.getName() and
-      func = getTwistedRequestHandlerMethod(name) and
-      this = func.getAReturnedNode()
-    )
-  }
-
-  override predicate sinks(TaintKind kind) { kind instanceof ExternalStringKind }
-
-  override string toString() { result = "Twisted response" }
-}
-
-/**
- * A sink of taint in the form of a "setter" method on a twisted request
- * object, which affects the properties of the subsequent response sent to this
- * request.
- */
-deprecated class TwistedRequestSetter extends HttpResponseTaintSink {
-  TwistedRequestSetter() {
-    exists(CallNode call, ControlFlowNode node, string name |
-      (
-        name = "setHeader" or
-        name = "addCookie" or
-        name = "write"
-      ) and
-      any(TwistedRequest t).taints(node) and
-      node = call.getFunction().(AttrNode).getObject(name) and
-      this = call.getAnArg()
-    )
-  }
-
-  override predicate sinks(TaintKind kind) { kind instanceof ExternalStringKind }
-
-  override string toString() { result = "Twisted request setter" }
-}