fix conflict

python/ql/lib/CHANGELOG.md

@@ -1,3 +1,28 @@
+## 1.0.1
+
+No user-facing changes.
+
+## 1.0.0
+
+### Breaking Changes
+
+* CodeQL package management is now generally available, and all GitHub-produced CodeQL packages have had their version numbers increased to 1.0.0.
+
+### New Features
+
+* A Python MaD (Models as Data) row may now contain a dotted path in the `type` column. Like in Ruby, a path to a class will refer to instances of that class. This means that the summary `["foo", "Member[MyClass].Instance.Member[instance_method]", "Argument[0]", "ReturnValue", "value"]` can now be written `["foo.MS_Class", "Member[instance_method]", "Argument[0]", "ReturnValue", "value"]`. To refer to an actual class, one may add a `!` at the end of the path.
+
+### Minor Analysis Improvements
+
+* The `request` parameter of Flask `SessionInterface.open_session` method is now modeled as a remote flow source.
+* Additional heuristics for a new sensitive data classification for private information (e.g. credit card numbers) have been added to the shared `SensitiveDataHeuristics.qll` library. This may result in additional results for queries that use sensitive data such as `py/clear-text-storage-sensitive-data` and `py/clear-text-logging-sensitive-data`.
+
+## 0.12.1
+
+### Major Analysis Improvements
+
+* Added modeling of the `pyramid` framework, leading to new remote flow sources and sinks.
+
 ## 0.12.0
 
 ### Breaking Changes

python/ql/lib/change-notes/released/0.12.1.md

@@ -0,0 +1,5 @@
+## 0.12.1
+
+### Major Analysis Improvements
+
+* Added modeling of the `pyramid` framework, leading to new remote flow sources and sinks.

python/ql/lib/change-notes/released/1.0.0.md

@@ -0,0 +1,14 @@
+## 1.0.0
+
+### Breaking Changes
+
+* CodeQL package management is now generally available, and all GitHub-produced CodeQL packages have had their version numbers increased to 1.0.0.
+
+### New Features
+
+* A Python MaD (Models as Data) row may now contain a dotted path in the `type` column. Like in Ruby, a path to a class will refer to instances of that class. This means that the summary `["foo", "Member[MyClass].Instance.Member[instance_method]", "Argument[0]", "ReturnValue", "value"]` can now be written `["foo.MS_Class", "Member[instance_method]", "Argument[0]", "ReturnValue", "value"]`. To refer to an actual class, one may add a `!` at the end of the path.
+
+### Minor Analysis Improvements
+
+* The `request` parameter of Flask `SessionInterface.open_session` method is now modeled as a remote flow source.
+* Additional heuristics for a new sensitive data classification for private information (e.g. credit card numbers) have been added to the shared `SensitiveDataHeuristics.qll` library. This may result in additional results for queries that use sensitive data such as `py/clear-text-storage-sensitive-data` and `py/clear-text-logging-sensitive-data`.

python/ql/lib/change-notes/released/1.0.1.md

@@ -0,0 +1,3 @@
+## 1.0.1
+
+No user-facing changes.

python/ql/lib/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.12.0
+lastReleaseVersion: 1.0.1

python/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/python-all
-version: 0.12.1-dev
+version: 1.0.2-dev
 groups: python
 dbscheme: semmlecode.python.dbscheme
 extractor: python

python/ql/lib/semmle/python/ApiGraphs.qll

@@ -226,6 +226,11 @@ module API {
      */
     Node getASubclass() { result = this.getASuccessor(Label::subclass()) }
 
+    /**
+     * Gets a node representing an instance of the class (or a transitive subclass of the class) represented by this node.
+     */
+    Node getAnInstance() { result = this.getASubclass*().getReturn() }
+
     /**
      * Gets a node representing the result from awaiting this node.
      */

python/ql/lib/semmle/python/Concepts.qll

@@ -200,10 +200,11 @@ module Decoding {
 }
 
 private class DecodingAdditionalTaintStep extends TaintTracking::AdditionalTaintStep {
-  override predicate step(DataFlow::Node nodeFrom, DataFlow::Node nodeTo) {
+  override predicate step(DataFlow::Node nodeFrom, DataFlow::Node nodeTo, string model) {
     exists(Decoding decoding |
       nodeFrom = decoding.getAnInput() and
-      nodeTo = decoding.getOutput()
+      nodeTo = decoding.getOutput() and
+      model = "Decoding-" + decoding.getFormat()
     )
   }
 }
@@ -1025,6 +1026,114 @@ module Http {
       }
     }
 
+    /**
+     * A data-flow node that sets a header in an HTTP response.
+     *
+     * Extend this class to model new APIs. If you want to refine existing API models,
+     * extend `ResponseHeaderWrite::Range` instead.
+     */
+    class ResponseHeaderWrite extends DataFlow::Node instanceof ResponseHeaderWrite::Range {
+      /**
+       * Gets the argument containing the header name.
+       */
+      DataFlow::Node getNameArg() { result = super.getNameArg() }
+
+      /**
+       * Gets the argument containing the header value.
+       */
+      DataFlow::Node getValueArg() { result = super.getValueArg() }
+
+      /**
+       * Holds if newlines are accepted in the header name argument.
+       */
+      predicate nameAllowsNewline() { super.nameAllowsNewline() }
+
+      /**
+       * Holds if newlines are accepted in the header value argument.
+       */
+      predicate valueAllowsNewline() { super.valueAllowsNewline() }
+    }
+
+    /** Provides a class for modeling header writes on HTTP responses. */
+    module ResponseHeaderWrite {
+      /**
+       *A data-flow node that sets a header in an HTTP response.
+       *
+       * Extend this class to model new APIs. If you want to refine existing API models,
+       * extend `ResponseHeaderWrite` instead.
+       */
+      abstract class Range extends DataFlow::Node {
+        /**
+         * Gets the argument containing the header name.
+         */
+        abstract DataFlow::Node getNameArg();
+
+        /**
+         * Gets the argument containing the header value.
+         */
+        abstract DataFlow::Node getValueArg();
+
+        /**
+         * Holds if newlines are accepted in the header name argument.
+         */
+        abstract predicate nameAllowsNewline();
+
+        /**
+         * Holds if newlines are accepted in the header value argument.
+         */
+        abstract predicate valueAllowsNewline();
+      }
+    }
+
+    /**
+     * A data-flow node that sets multiple headers in an HTTP response using a dict or a list of tuples.
+     *
+     * Extend this class to model new APIs. If you want to refine existing API models,
+     * extend `ResponseHeaderBulkWrite::Range` instead.
+     */
+    class ResponseHeaderBulkWrite extends DataFlow::Node instanceof ResponseHeaderBulkWrite::Range {
+      /**
+       * Gets the argument containing the headers dictionary.
+       */
+      DataFlow::Node getBulkArg() { result = super.getBulkArg() }
+
+      /**
+       * Holds if newlines are accepted in the header name argument.
+       */
+      predicate nameAllowsNewline() { super.nameAllowsNewline() }
+
+      /**
+       * Holds if newlines are accepted in the header value argument.
+       */
+      predicate valueAllowsNewline() { super.valueAllowsNewline() }
+    }
+
+    /** Provides a class for modeling bulk header writes on HTTP responses. */
+    module ResponseHeaderBulkWrite {
+      /**
+       * A data-flow node that sets multiple headers in an HTTP response using a dict.
+       *
+       * Extend this class to model new APIs. If you want to refine existing API models,
+       * extend `ResponseHeaderBulkWrite` instead.
+       */
+      abstract class Range extends DataFlow::Node {
+        /**
+         * Gets the argument containing the headers dictionary.
+         */
+        abstract DataFlow::Node getBulkArg();
+
+        /**
+         * Holds if newlines are accepted in the header name argument.
+         */
+        abstract predicate nameAllowsNewline();
+
+        /**
+         * Holds if newlines are accepted in the header value argument.
+         */
+        abstract predicate valueAllowsNewline();
+      }
+    }
+
     /**
      * A data-flow node that sets a cookie in an HTTP response.
      *

python/ql/lib/semmle/python/Frameworks.qll

@@ -29,6 +29,7 @@ private import semmle.python.frameworks.FastApi
 private import semmle.python.frameworks.Flask
 private import semmle.python.frameworks.FlaskAdmin
 private import semmle.python.frameworks.FlaskSqlAlchemy
+private import semmle.python.frameworks.Gradio
 private import semmle.python.frameworks.Httpx
 private import semmle.python.frameworks.Idna
 private import semmle.python.frameworks.Invoke
@@ -45,6 +46,7 @@ private import semmle.python.frameworks.Multidict
 private import semmle.python.frameworks.Mysql
 private import semmle.python.frameworks.MySQLdb
 private import semmle.python.frameworks.Numpy
+private import semmle.python.frameworks.Opml
 private import semmle.python.frameworks.Oracledb
 private import semmle.python.frameworks.Pandas
 private import semmle.python.frameworks.Paramiko
@@ -59,6 +61,7 @@ private import semmle.python.frameworks.PyMongo
 private import semmle.python.frameworks.Pymssql
 private import semmle.python.frameworks.PyMySQL
 private import semmle.python.frameworks.Pyodbc
+private import semmle.python.frameworks.Pyramid
 private import semmle.python.frameworks.Requests
 private import semmle.python.frameworks.RestFramework
 private import semmle.python.frameworks.Rsa

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowDispatch.qll

@@ -344,6 +344,16 @@ abstract class DataFlowCallable extends TDataFlowCallable {
 
   /** Gets the location of this dataflow callable. */
   abstract Location getLocation();
+
+  /** Gets a best-effort total ordering. */
+  int totalorder() {
+    this =
+      rank[result](DataFlowCallable c, string file, int startline, int startcolumn |
+        c.getLocation().hasLocationInfo(file, startline, startcolumn, _, _)
+      |
+        c order by file, startline, startcolumn
+      )
+  }
 }
 
 /** A callable function. */
@@ -1419,6 +1429,16 @@ abstract class DataFlowCall extends TDataFlowCall {
   ) {
     this.getLocation().hasLocationInfo(filepath, startline, startcolumn, endline, endcolumn)
   }
+
+  /** Gets a best-effort total ordering. */
+  int totalorder() {
+    this =
+      rank[result](DataFlowCall c, int startline, int startcolumn |
+        c.hasLocationInfo(_, startline, startcolumn, _, _)
+      |
+        c order by startline, startcolumn
+      )
+  }
 }
 
 /** A call found in the program source (as opposed to a synthesised call). */

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowPrivate.qll

@@ -1023,13 +1023,21 @@ predicate attributeClearStep(Node n, AttributeContent c) {
   exists(PostUpdateNode post | post.getPreUpdateNode() = n | attributeStoreStep(_, c, post))
 }
 
+class NodeRegion instanceof Unit {
+  string toString() { result = "NodeRegion" }
+
+  predicate contains(Node n) { none() }
+
+  int totalOrder() { result = 1 }
+}
+
 //--------
 // Fancy context-sensitive guards
 //--------
 /**
- * Holds if the node `n` is unreachable when the call context is `call`.
+ * Holds if the nodes in `nr` are unreachable when the call context is `call`.
  */
-predicate isUnreachableInCall(Node n, DataFlowCall call) { none() }
+predicate isUnreachableInCall(NodeRegion nr, DataFlowCall call) { none() }
 
 /**
  * Holds if access paths with `c` at their head always should be tracked at high

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowPublic.qll

@@ -9,6 +9,7 @@ import Attributes
 import LocalSources
 private import semmle.python.essa.SsaCompute
 private import semmle.python.dataflow.new.internal.ImportStar
+private import semmle.python.frameworks.data.ModelsAsData
 private import FlowSummaryImpl as FlowSummaryImpl
 private import semmle.python.frameworks.data.ModelsAsData
 
@@ -125,6 +126,13 @@ newtype TNode =
     f = any(VariableCapture::CapturedVariable v).getACapturingScope() and
     // TODO: Remove this restriction when adding proper support for captured variables in the body of the function we generate for comprehensions
     exists(TFunction(f))
+  } or
+  /** An empty, unused node type that exists to prevent unwanted dependencies on data flow nodes. */
+  TForbiddenRecursionGuard() {
+    none() and
+    // We want to prune irrelevant models before materialising data flow nodes, so types contributed
+    // directly from CodeQL must expose their pruning info without depending on data flow nodes.
+    (any(ModelInput::TypeModel tm).isTypeUsed("") implies any())
   }
 
 private import semmle.python.internal.CachedStages

python/ql/lib/semmle/python/dataflow/new/internal/TaintTrackingPrivate.qll

@@ -27,8 +27,7 @@ private module Cached {
   predicate defaultAdditionalTaintStep(DataFlow::Node nodeFrom, DataFlow::Node nodeTo, string model) {
     localAdditionalTaintStep(nodeFrom, nodeTo, model)
     or
-    any(AdditionalTaintStep a).step(nodeFrom, nodeTo) and
-    model = "AdditionalTaintStep"
+    any(AdditionalTaintStep a).hasStep(nodeFrom, nodeTo, model)
   }
 
   /**

python/ql/lib/semmle/python/dataflow/new/internal/TaintTrackingPublic.qll

@@ -47,6 +47,25 @@ class AdditionalTaintStep extends Unit {
   /**
    * Holds if the step from `nodeFrom` to `nodeTo` should be considered a taint
    * step for all configurations.
+   *
+   * Note that it is now possible to also specify provenance of the taint step
+   * by overwriting `step/3`.
    */
-  abstract predicate step(DataFlow::Node nodeFrom, DataFlow::Node nodeTo);
+  predicate step(DataFlow::Node nodeFrom, DataFlow::Node nodeTo) { none() }
+
+  /**
+   * Holds if the step from `nodeFrom` to `nodeTo` should be considered a taint
+   * step with provenance `model` for all configurations.
+   */
+  predicate step(DataFlow::Node nodeFrom, DataFlow::Node nodeTo, string model) { none() }
+
+  /**
+   * Holds if this `AdditionalTaintStep` defines a step from `nodeFrom` to `nodeTo`
+   * with provenance `model`.
+   */
+  final predicate hasStep(DataFlow::Node nodeFrom, DataFlow::Node nodeTo, string model) {
+    this.step(nodeFrom, nodeTo) and model = "AdditionalTaintStep"
+    or
+    this.step(nodeFrom, nodeTo, model)
+  }
 }

python/ql/lib/semmle/python/frameworks/Flask.qll

@@ -101,6 +101,19 @@ module Flask {
   /** Gets a reference to the `flask.request` object. */
   API::Node request() {
     result = API::moduleImport(["flask", "flask_restful"]).getMember("request")
+    or
+    result = sessionInterfaceRequestParam()
+  }
+
+  /** Gets a `request` parameter of an implementation of `open_session` in a subclass of `flask.sessions.SessionInterface` */
+  private API::Node sessionInterfaceRequestParam() {
+    result =
+      API::moduleImport("flask")
+          .getMember("sessions")
+          .getMember("SessionInterface")
+          .getASubclass+()
+          .getMember("open_session")
+          .getParameter(1)
   }
 
   /**
@@ -220,6 +233,43 @@ module Flask {
 
     /** Gets a reference to an instance of `flask.Response`. */
     DataFlow::Node instance() { instance(DataFlow::TypeTracker::end()).flowsTo(result) }
+
+    /** An `Headers` instance that is part of a Flask response. */
+    private class FlaskResponseHeadersInstances extends Werkzeug::Headers::InstanceSource {
+      FlaskResponseHeadersInstances() {
+        this.(DataFlow::AttrRead).getObject() = instance() and
+        this.(DataFlow::AttrRead).getAttributeName() = "headers"
+      }
+    }
+
+    /** A class instantiation of `Response` that sets response headers. */
+    private class ResponseClassHeadersWrite extends Http::Server::ResponseHeaderBulkWrite::Range,
+      ClassInstantiation
+    {
+      override DataFlow::Node getBulkArg() {
+        result = [this.getArg(2), this.getArgByName("headers")]
+      }
+
+      override predicate nameAllowsNewline() { any() }
+
+      override predicate valueAllowsNewline() { none() }
+    }
+
+    /** A call to `make_response that sets response headers. */
+    private class MakeResponseHeadersWrite extends Http::Server::ResponseHeaderBulkWrite::Range,
+      FlaskMakeResponseCall
+    {
+      override DataFlow::Node getBulkArg() {
+        result = this.getArg(2)
+        or
+        strictcount(this.getArg(_)) = 2 and
+        result = this.getArg(1)
+      }
+
+      override predicate nameAllowsNewline() { any() }
+
+      override predicate valueAllowsNewline() { none() }
+    }
   }
 
   // ---------------------------------------------------------------------------

python/ql/lib/semmle/python/frameworks/Gradio.qll

@@ -0,0 +1,123 @@
+/**
+ * Provides classes modeling security-relevant aspects of the `gradio` PyPI package.
+ * See https://pypi.org/project/gradio/.
+ */
+
+import python
+import semmle.python.dataflow.new.RemoteFlowSources
+import semmle.python.dataflow.new.TaintTracking
+import semmle.python.ApiGraphs
+
+/**
+ * Provides models for the `gradio` PyPI package.
+ * See https://pypi.org/project/gradio/.
+ */
+module Gradio {
+  /**
+   * The event handlers, Interface and gradio.ChatInterface classes, which take untrusted data.
+   */
+  private class GradioInput extends API::CallNode {
+    GradioInput() {
+      this =
+        API::moduleImport("gradio")
+            .getMember([
+                "Button", "Textbox", "UploadButton", "Slider", "JSON", "HTML", "Markdown", "File",
+                "AnnotatedImage", "Audio", "BarPlot", "Chatbot", "Checkbox", "CheckboxGroup",
+                "ClearButton", "Code", "ColorPicker", "Dataframe", "Dataset", "DownloadButton",
+                "Dropdown", "DuplicateButton", "FileExplorer", "Gallery", "HighlightedText",
+                "Image", "ImageEditor", "Label", "LinePlot", "LoginButton", "LogoutButton",
+                "Model3D", "Number", "ParamViewer", "Plot", "Radio", "ScatterPlot", "SimpleImage",
+                "State", "Video"
+              ])
+            .getReturn()
+            .getMember([
+                "change", "input", "click", "submit", "edit", "clear", "play", "pause", "stop",
+                "end", "start_recording", "pause_recording", "stop_recording", "focus", "blur",
+                "upload", "release", "select", "stream", "like", "load", "key_up",
+              ])
+            .getACall()
+      or
+      this = API::moduleImport("gradio").getMember(["Interface", "ChatInterface"]).getACall()
+    }
+  }
+
+  /**
+   * The `inputs` parameters in Gradio event handlers, that are lists and are sources of untrusted data.
+   * This model allows tracking each element list back to source, f.ex. `gr.Textbox(...)`.
+   */
+  private class GradioInputList extends RemoteFlowSource::Range {
+    GradioInputList() {
+      exists(GradioInput call |
+        // limit only to lists of parameters given to `inputs`.
+        (
+          (
+            call.getKeywordParameter("inputs").asSink().asCfgNode() instanceof ListNode
+            or
+            call.getParameter(1).asSink().asCfgNode() instanceof ListNode
+          ) and
+          (
+            this = call.getKeywordParameter("inputs").getASubscript().getAValueReachingSink()
+            or
+            this = call.getParameter(1).getASubscript().getAValueReachingSink()
+          )
+        )
+      )
+    }
+
+    override string getSourceType() { result = "Gradio untrusted input" }
+  }
+
+  /**
+   * The `inputs` parameters in Gradio event handlers, that are not lists and are sources of untrusted data.
+   */
+  private class GradioInputParameter extends RemoteFlowSource::Range {
+    GradioInputParameter() {
+      exists(GradioInput call |
+        this = call.getParameter(0, "fn").getParameter(_).asSource() and
+        // exclude lists of parameters given to `inputs`
+        not call.getKeywordParameter("inputs").asSink().asCfgNode() instanceof ListNode and
+        not call.getParameter(1).asSink().asCfgNode() instanceof ListNode
+      )
+    }
+
+    override string getSourceType() { result = "Gradio untrusted input" }
+  }
+
+  /**
+   * The `inputs` parameters in Gradio decorators to event handlers, that are sources of untrusted data.
+   */
+  private class GradioInputDecorator extends RemoteFlowSource::Range {
+    GradioInputDecorator() {
+      exists(GradioInput call |
+        this = call.getReturn().getACall().getParameter(0).getParameter(_).asSource()
+      )
+    }
+
+    override string getSourceType() { result = "Gradio untrusted input" }
+  }
+
+  /**
+   * Extra taint propagation for tracking `inputs` parameters in Gradio event handlers, that are lists.
+   */
+  private class ListTaintStep extends TaintTracking::AdditionalTaintStep {
+    override predicate step(DataFlow::Node nodeFrom, DataFlow::Node nodeTo) {
+      exists(GradioInput node |
+        // handle cases where there are multiple arguments passed as a list to `inputs`
+        (
+          (
+            node.getKeywordParameter("inputs").asSink().asCfgNode() instanceof ListNode
+            or
+            node.getParameter(1).asSink().asCfgNode() instanceof ListNode
+          ) and
+          exists(int i | nodeTo = node.getParameter(0, "fn").getParameter(i).asSource() |
+            nodeFrom.asCfgNode() =
+              node.getKeywordParameter("inputs").asSink().asCfgNode().(ListNode).getElement(i)
+            or
+            nodeFrom.asCfgNode() =
+              node.getParameter(1).asSink().asCfgNode().(ListNode).getElement(i)
+          )
+        )
+      )
+    }
+  }
+}

python/ql/lib/semmle/python/frameworks/Opml.qll

@@ -0,0 +1,64 @@
+/**
+ * Provides classes modeling security-relevant aspects of the `opml` PyPI package.
+ *
+ * See
+ * - https://pypi.org/project/opml/
+ */
+
+private import python
+private import semmle.python.dataflow.new.DataFlow
+private import semmle.python.Concepts
+private import semmle.python.ApiGraphs
+
+/**
+ * Provides classes modeling security-relevant aspects of the `opml` PyPI package
+ *
+ * See
+ * - https://pypi.org/project/opml/
+ */
+private module Opml {
+  /**
+   * A call to the `xpath` method of a parsed document.
+   *
+   *    import opml
+   *    root = opml.from_string(file(XML_DB).read())
+   *    find_text = root.xpath("`sink`")
+   */
+  private class XPathCall extends XML::XPathExecution::Range, DataFlow::CallCfgNode {
+    XPathCall() {
+      exists(API::Node parseResult |
+        parseResult = API::moduleImport("opml").getMember(["parse", "from_string"]).getReturn()
+      |
+        this = parseResult.getMember("xpath").getACall()
+      )
+    }
+
+    override DataFlow::Node getXPath() { result = this.getArg(0) }
+
+    override string getName() { result = "opml" }
+  }
+
+  /**
+   * A call to either of:
+   * - `opml.parse`
+   * - `opml.from_string`
+   */
+  private class OpmlParsing extends DataFlow::CallCfgNode, XML::XmlParsing::Range {
+    OpmlParsing() {
+      this = API::moduleImport("opml").getMember(["parse", "from_string"]).getACall()
+    }
+
+    override DataFlow::Node getAnInput() { result = this.getArg(0) }
+
+    DataFlow::Node getParserArg() { none() }
+
+    /**
+     * The same as `Lxml::LxmlParsing::vulnerableTo`, because `opml` uses `lxml` for parsing.
+     */
+    override predicate vulnerableTo(XML::XmlParsingVulnerabilityKind kind) { kind.isXxe() }
+
+    override predicate mayExecuteInput() { none() }
+
+    override DataFlow::Node getOutput() { result = this }
+  }
+}

python/ql/lib/semmle/python/frameworks/Pyramid.qll

@@ -0,0 +1,299 @@
+/**
+ * Provides classes modeling security-relevant aspects of the `pyramid` PyPI package.
+ * See https://docs.pylonsproject.org/projects/pyramid/.
+ */
+
+private import python
+private import semmle.python.dataflow.new.RemoteFlowSources
+private import semmle.python.dataflow.new.TaintTracking
+private import semmle.python.Concepts
+private import semmle.python.ApiGraphs
+private import semmle.python.dataflow.new.FlowSummary
+private import semmle.python.frameworks.internal.PoorMansFunctionResolution
+private import semmle.python.frameworks.internal.InstanceTaintStepsHelper
+private import semmle.python.frameworks.data.ModelsAsData
+private import semmle.python.frameworks.Stdlib
+
+/**
+ * Provides models for the `pyramid` PyPI package.
+ * See https://docs.pylonsproject.org/projects/pyramid/.
+ */
+module Pyramid {
+  /** Provides models for pyramid View callables. */
+  module View {
+    /** A dataflow node that sets up a route on a server using the Pyramid framework. */
+    abstract private class PyramidRouteSetup extends Http::Server::RouteSetup::Range {
+      override string getFramework() { result = "Pyramid" }
+    }
+
+    /**
+     * A Pyramid view callable, that handles incoming requests.
+     */
+    class ViewCallable extends Function {
+      ViewCallable() { this = any(PyramidRouteSetup rs).getARequestHandler() }
+
+      /** Gets the `request` parameter of this callable. */
+      Parameter getRequestParameter() {
+        this.getPositionalParameterCount() = 1 and
+        result = this.getArg(0)
+        or
+        this.getPositionalParameterCount() = 2 and
+        result = this.getArg(1)
+      }
+    }
+
+    /** A pyramid route setup using the `pyramid.view.view_config` decorator. */
+    private class DecoratorSetup extends PyramidRouteSetup {
+      DecoratorSetup() {
+        this = API::moduleImport("pyramid").getMember("view").getMember("view_config").getACall()
+      }
+
+      override Function getARequestHandler() { result.getADecorator() = this.asExpr() }
+
+      override DataFlow::Node getUrlPatternArg() { none() } // there is a `route_name` arg, but that does not contain the url pattern
+
+      override Parameter getARoutedParameter() { none() }
+    }
+
+    /** A pyramid route setup using a call to `pyramid.config.Configurator.add_view`. */
+    private class ConfiguratorSetup extends PyramidRouteSetup instanceof Configurator::AddViewCall {
+      override Function getARequestHandler() {
+        this.(Configurator::AddViewCall).getViewArg() = poorMansFunctionTracker(result)
+      }
+
+      override DataFlow::Node getUrlPatternArg() { none() } // there is a `route_name` arg, but that does not contain the url pattern
+
+      override Parameter getARoutedParameter() { none() }
+    }
+  }
+
+  /** Provides models for `pyramid.config.Configurator` */
+  module Configurator {
+    /** Gets a reference to the class `pyramid.config.Configurator`. */
+    API::Node classRef() {
+      result = API::moduleImport("pyramid").getMember("config").getMember("Configurator")
+    }
+
+    /** Gets a reference to an instance of `pyramid.config.Configurator`. */
+    private DataFlow::TypeTrackingNode instance(DataFlow::TypeTracker t) {
+      t.start() and
+      result = classRef().getACall()
+      or
+      exists(DataFlow::TypeTracker t2 | result = instance(t2).track(t2, t))
+    }
+
+    /** Gets a reference to an instance of `pyramid.config.Configurator`. */
+    DataFlow::Node instance() { instance(DataFlow::TypeTracker::end()).flowsTo(result) }
+
+    /** A call to the `add_view` method of an instance of `pyramid.config.Configurator`. */
+    class AddViewCall extends DataFlow::MethodCallNode {
+      AddViewCall() { this.calls(instance(), "add_view") }
+
+      /** Gets the `view` argument of this call. */
+      DataFlow::Node getViewArg() { result = [this.getArg(0), this.getArgByName("view")] }
+    }
+  }
+
+  /** Provides modeling for pyramid requests. */
+  module Request {
+    /**
+     * A source of instances of `pyramid.request.Request`, extend this class to model new instances.
+     *
+     * Use the predicate `Request::instance()` to get references to instances of `pyramid.request.Request`.
+     */
+    abstract class InstanceSource extends DataFlow::LocalSourceNode { }
+
+    /** Gets a reference to an instance of `pyramid.request.Request`. */
+    private DataFlow::TypeTrackingNode instance(DataFlow::TypeTracker t) {
+      t.start() and
+      result instanceof InstanceSource
+      or
+      exists(DataFlow::TypeTracker t2 | result = instance(t2).track(t2, t))
+    }
+
+    /** Gets a reference to an instance of `pyramid.request.Request`. */
+    DataFlow::Node instance() { instance(DataFlow::TypeTracker::end()).flowsTo(result) }
+
+    private class RequestParameter extends InstanceSource, RemoteFlowSource::Range instanceof DataFlow::ParameterNode
+    {
+      RequestParameter() { this.getParameter() = any(View::ViewCallable vc).getRequestParameter() }
+
+      override string getSourceType() { result = "Pyramid request parameter" }
+    }
+
+    /** Taint steps for request instances. */
+    private class InstanceTaintSteps extends InstanceTaintStepsHelper {
+      InstanceTaintSteps() { this = "pyramid.request.Request" }
+
+      override DataFlow::Node getInstance() { result = instance() }
+
+      override string getAttributeName() {
+        result in [
+            "accept", "accept_charset", "accept_encoding", "accept_language", "application_url",
+            "as_bytes", "authorization", "body", "body_file", "body_file_raw", "body_file_seekable",
+            "cache_control", "client_addr", "content_type", "cookies", "domain", "headers", "host",
+            "host_port", "host_url", "GET", "if_match", "if_none_match", "if_range",
+            "if_none_match", "json", "json_body", "matchdict", "params", "path", "path_info",
+            "path_qs", "path_url", "POST", "pragma", "query_string", "range", "referer", "referrer",
+            "text", "url", "urlargs", "urlvars", "user_agent"
+          ]
+      }
+
+      override string getMethodName() {
+        result in ["as_bytes", "copy", "copy_get", "path_info_peek", "path_info_pop"]
+      }
+
+      override string getAsyncMethodName() { none() }
+    }
+
+    /** A call to a method of a `request` that copies the request. */
+    private class RequestCopyCall extends InstanceSource, DataFlow::MethodCallNode {
+      RequestCopyCall() { this.calls(instance(), ["copy", "copy_get"]) }
+    }
+
+    /** A member of a request that is a file-like object. */
+    private class RequestBodyFileLike extends Stdlib::FileLikeObject::InstanceSource instanceof DataFlow::AttrRead
+    {
+      RequestBodyFileLike() {
+        this.getObject() = instance() and
+        this.getAttributeName() = ["body_file", "body_file_raw", "body_file_seekable"]
+      }
+    }
+  }
+
+  /** Provides modeling for pyramid responses. */
+  module Response {
+    /** A response returned by a view callable. */
+    private class PyramidReturnResponse extends Http::Server::HttpResponse::Range {
+      PyramidReturnResponse() {
+        this.asCfgNode() = any(View::ViewCallable vc).getAReturnValueFlowNode() and
+        not this = instance()
+      }
+
+      override DataFlow::Node getBody() { result = this }
+
+      override DataFlow::Node getMimetypeOrContentTypeArg() { none() }
+
+      override string getMimetypeDefault() { result = "text/html" }
+    }
+
+    /** Gets a reference to the class `pyramid.response.Response`. */
+    API::Node classRef() {
+      result = API::moduleImport("pyramid").getMember("response").getMember("Response")
+    }
+
+    /**
+     * A source of instances of `pyramid.response.Response`, extend this class to model new instances.
+     *
+     * This can include instantiations of the class, return values from function
+     * calls, or a special parameter that will be set when functions are called by an external
+     * library.
+     *
+     * Use the predicate `Response::instance()` to get references to instances of `pyramid.response.Response`.
+     */
+    abstract class InstanceSource extends DataFlow::LocalSourceNode,
+      Http::Server::HttpResponse::Range
+    { }
+
+    /** Gets a reference to an instance of `pyramid.response.Response`. */
+    private DataFlow::TypeTrackingNode instance(DataFlow::TypeTracker t) {
+      t.start() and
+      result instanceof InstanceSource
+      or
+      exists(DataFlow::TypeTracker t2 | result = instance(t2).track(t2, t))
+    }
+
+    /** Gets a reference to an instance of `pyramid.response.Response`. */
+    DataFlow::Node instance() { instance(DataFlow::TypeTracker::end()).flowsTo(result) }
+
+    /** An instantiation of the class `pyramid.response.Response` or a subclass. */
+    private class ClassInstantiation extends InstanceSource, DataFlow::CallCfgNode {
+      ClassInstantiation() { this = classRef().getACall() }
+
+      override DataFlow::Node getBody() { result = [this.getArg(0), this.getArgByName("body")] }
+
+      override DataFlow::Node getMimetypeOrContentTypeArg() {
+        result = [this.getArg(4), this.getArgByName("content_type")]
+      }
+
+      override string getMimetypeDefault() { result = "text/html" }
+    }
+
+    /** A write to a field that sets the body of a response. */
+    private class ResponseBodySet extends Http::Server::HttpResponse::Range instanceof DataFlow::AttrWrite
+    {
+      string attrName;
+
+      ResponseBodySet() {
+        this.getObject() = instance() and
+        this.getAttributeName() = attrName and
+        attrName in ["body", "body_file", "json", "json_body", "text", "ubody", "unicode_body"]
+      }
+
+      override DataFlow::Node getBody() { result = this.(DataFlow::AttrWrite).getValue() }
+
+      override DataFlow::Node getMimetypeOrContentTypeArg() { none() }
+
+      override string getMimetypeDefault() {
+        if attrName in ["json", "json_body"]
+        then result = "application/json"
+        else result = "text/html"
+      }
+    }
+
+    /** A use of the `response` attribute of a `Request`. */
+    private class RequestResponseAttr extends InstanceSource instanceof DataFlow::AttrRead {
+      RequestResponseAttr() {
+        this.getObject() = Request::instance() and this.getAttributeName() = "response"
+      }
+
+      override DataFlow::Node getBody() { none() }
+
+      override DataFlow::Node getMimetypeOrContentTypeArg() { none() }
+
+      override string getMimetypeDefault() { result = "text/html" }
+    }
+
+    /** A call to `response.set_cookie`. */
+    private class SetCookieCall extends Http::Server::CookieWrite::Range, DataFlow::MethodCallNode {
+      SetCookieCall() { this.calls(instance(), "set_cookie") }
+
+      override DataFlow::Node getHeaderArg() { none() }
+
+      override DataFlow::Node getNameArg() { result = [this.getArg(0), this.getArgByName("name")] }
+
+      override DataFlow::Node getValueArg() {
+        result = [this.getArg(1), this.getArgByName("value")]
+      }
+    }
+  }
+
+  /** Provides models for pyramid http redirects. */
+  module Redirect {
+    /** Gets a reference to a class that represents an HTTP redirect response.. */
+    API::Node classRef() {
+      result =
+        API::moduleImport("pyramid")
+            .getMember("httpexceptions")
+            .getMember([
+                "HTTPMultipleChoices", "HTTPMovedPermanently", "HTTPFound", "HTTPSeeOther",
+                "HTTPUseProxy", "HTTPTemporaryRedirect", "HTTPPermanentRedirect"
+              ])
+    }
+
+    /** A call to a pyramid HTTP exception class that represents an HTTP redirect response. */
+    class PyramidRedirect extends Http::Server::HttpRedirectResponse::Range, DataFlow::CallCfgNode {
+      PyramidRedirect() { this = classRef().getACall() }
+
+      override DataFlow::Node getRedirectLocation() {
+        result = [this.getArg(0), this.getArgByName("location")]
+      }
+
+      override DataFlow::Node getBody() { none() }
+
+      override DataFlow::Node getMimetypeOrContentTypeArg() { none() }
+
+      override string getMimetypeDefault() { result = "text/html" }
+    }
+  }
+}

python/ql/lib/semmle/python/frameworks/Stdlib.qll

@@ -2183,17 +2183,35 @@ module StdlibPrivate {
      * for how a request is processed and given to an application.
      */
     class WsgirefSimpleServerApplication extends Http::Server::RequestHandler::Range {
+      boolean validator;
+
       WsgirefSimpleServerApplication() {
         exists(DataFlow::Node appArg, DataFlow::CallCfgNode setAppCall |
           (
             setAppCall =
-              WsgirefSimpleServer::subclassRef().getReturn().getMember("set_app").getACall()
+              WsgirefSimpleServer::subclassRef().getReturn().getMember("set_app").getACall() and
+            validator = false
             or
             setAppCall
                 .(DataFlow::MethodCallNode)
-                .calls(any(WsgiServerSubclass cls).getASelfRef(), "set_app")
+                .calls(any(WsgiServerSubclass cls).getASelfRef(), "set_app") and
+            validator = false
+            or
+            // assume an application that is passed to `wsgiref.validate.validator` is eventually passed to `set_app`
+            setAppCall =
+              API::moduleImport("wsgiref").getMember("validate").getMember("validator").getACall() and
+            validator = true
           ) and
           appArg in [setAppCall.getArg(0), setAppCall.getArgByName("application")]
+          or
+          // `make_server` calls `set_app`
+          setAppCall =
+            API::moduleImport("wsgiref")
+                .getMember("simple_server")
+                .getMember("make_server")
+                .getACall() and
+          appArg in [setAppCall.getArg(2), setAppCall.getArgByName("app")] and
+          validator = false
         |
           appArg = poorMansFunctionTracker(this)
         )
@@ -2202,6 +2220,9 @@ module StdlibPrivate {
       override Parameter getARoutedParameter() { none() }
 
       override string getFramework() { result = "Stdlib: wsgiref.simple_server application" }
+
+      /** Holds if this simple server application was passed to `wsgiref.validate.validator`. */
+      predicate isValidated() { validator = true }
     }
 
     /**
@@ -2305,6 +2326,114 @@ module StdlibPrivate {
 
       override string getMimetypeDefault() { none() }
     }
+
+    /**
+     * Provides models for the `wsgiref.headers.Headers` class
+     *
+     * See https://docs.python.org/3/library/wsgiref.html#module-wsgiref.headers.
+     */
+    module Headers {
+      /** Gets a reference to the `wsgiref.headers.Headers` class. */
+      API::Node classRef() {
+        result = API::moduleImport("wsgiref").getMember("headers").getMember("Headers")
+        or
+        result = ModelOutput::getATypeNode("wsgiref.headers.Headers~Subclass").getASubclass*()
+      }
+
+      /** Gets a reference to an instance of `wsgiref.headers.Headers`. */
+      private DataFlow::TypeTrackingNode instance(DataFlow::TypeTracker t) {
+        t.start() and
+        result = classRef().getACall()
+        or
+        exists(DataFlow::TypeTracker t2 | result = instance(t2).track(t2, t))
+      }
+
+      /** Gets a reference to an instance of `wsgiref.headers.Headers`. */
+      DataFlow::Node instance() { instance(DataFlow::TypeTracker::end()).flowsTo(result) }
+
+      /** Holds if there exists an application that is validated by `wsgiref.validate.validator`. */
+      private predicate existsValidatedApplication() {
+        exists(WsgirefSimpleServerApplication app | app.isValidated())
+      }
+
+      /** A class instantiation of `wsgiref.headers.Headers`, conidered as a write to a response header. */
+      private class WsgirefHeadersInstantiation extends Http::Server::ResponseHeaderBulkWrite::Range,
+        DataFlow::CallCfgNode
+      {
+        WsgirefHeadersInstantiation() { this = classRef().getACall() }
+
+        override DataFlow::Node getBulkArg() {
+          result = [this.getArg(0), this.getArgByName("headers")]
+        }
+
+        // TODO: These checks perhaps could be made more precise.
+        override predicate nameAllowsNewline() { not existsValidatedApplication() }
+
+        override predicate valueAllowsNewline() { not existsValidatedApplication() }
+      }
+
+      /** A call to a method that writes to a response header. */
+      private class HeaderWriteCall extends Http::Server::ResponseHeaderWrite::Range,
+        DataFlow::MethodCallNode
+      {
+        HeaderWriteCall() {
+          this.calls(instance(), ["add_header", "set", "setdefault", "__setitem__"])
+        }
+
+        override DataFlow::Node getNameArg() { result = this.getArg(0) }
+
+        override DataFlow::Node getValueArg() { result = this.getArg(1) }
+
+        // TODO: These checks perhaps could be made more precise.
+        override predicate nameAllowsNewline() { not existsValidatedApplication() }
+
+        override predicate valueAllowsNewline() { not existsValidatedApplication() }
+      }
+
+      /** A dict-like write to a response header. */
+      private class HeaderWriteSubscript extends Http::Server::ResponseHeaderWrite::Range,
+        DataFlow::Node
+      {
+        DataFlow::Node name;
+        DataFlow::Node value;
+
+        HeaderWriteSubscript() {
+          exists(SubscriptNode subscript |
+            this.asCfgNode() = subscript and
+            value.asCfgNode() = subscript.(DefinitionNode).getValue() and
+            name.asCfgNode() = subscript.getIndex() and
+            subscript.getObject() = instance().asCfgNode()
+          )
+        }
+
+        override DataFlow::Node getNameArg() { result = name }
+
+        override DataFlow::Node getValueArg() { result = value }
+
+        // TODO: These checks perhaps could be made more precise.
+        override predicate nameAllowsNewline() { not existsValidatedApplication() }
+
+        override predicate valueAllowsNewline() { not existsValidatedApplication() }
+      }
+
+      /**
+       * A call to a `start_response` function that sets the response headers.
+       */
+      private class WsgirefSimpleServerSetHeaders extends Http::Server::ResponseHeaderBulkWrite::Range,
+        DataFlow::CallCfgNode
+      {
+        WsgirefSimpleServerSetHeaders() { this.getFunction() = startResponse() }
+
+        override DataFlow::Node getBulkArg() {
+          result = [this.getArg(1), this.getArgByName("headers")]
+        }
+
+        // TODO: These checks perhaps could be made more precise.
+        override predicate nameAllowsNewline() { not existsValidatedApplication() }
+
+        override predicate valueAllowsNewline() { not existsValidatedApplication() }
+      }
+    }
   }
 
   // ---------------------------------------------------------------------------

python/ql/lib/semmle/python/frameworks/Werkzeug.qll

@@ -12,6 +12,7 @@ private import semmle.python.ApiGraphs
 private import semmle.python.frameworks.Stdlib
 private import semmle.python.Concepts
 private import semmle.python.frameworks.internal.InstanceTaintStepsHelper
+private import semmle.python.frameworks.data.ModelsAsData
 
 /**
  * Provides models for the `Werkzeug` PyPI package.
@@ -144,6 +145,18 @@ module Werkzeug {
    * See https://werkzeug.palletsprojects.com/en/1.0.x/datastructures/#werkzeug.datastructures.Headers.
    */
   module Headers {
+    /** Gets a reference to the `werkzeug.datastructures.Headers` class. */
+    API::Node classRef() {
+      result = API::moduleImport("werkzeug").getMember("datastructures").getMember("Headers")
+      or
+      result = ModelOutput::getATypeNode("werkzeug.datastructures.Headers~Subclass").getASubclass*()
+    }
+
+    /** A direct instantiation of `werkzeug.datastructures.Headers`. */
+    private class ClassInstantiation extends InstanceSource, DataFlow::CallCfgNode {
+      ClassInstantiation() { this = classRef().getACall() }
+    }
+
     /**
      * A source of instances of `werkzeug.datastructures.Headers`, extend this class to model new instances.
      *
@@ -182,6 +195,61 @@ module Werkzeug {
 
       override string getAsyncMethodName() { none() }
     }
+
+    /** A call to a method that writes to a header, assumed to be a response header. */
+    private class HeaderWriteCall extends Http::Server::ResponseHeaderWrite::Range,
+      DataFlow::MethodCallNode
+    {
+      HeaderWriteCall() {
+        this.calls(instance(), ["add", "add_header", "set", "setdefault", "__setitem__"])
+      }
+
+      override DataFlow::Node getNameArg() { result = this.getArg(0) }
+
+      override DataFlow::Node getValueArg() { result = this.getArg(1) }
+
+      override predicate nameAllowsNewline() { any() }
+
+      override predicate valueAllowsNewline() { none() }
+    }
+
+    /** A dict-like write to a header, assumed to be a response header. */
+    private class HeaderWriteSubscript extends Http::Server::ResponseHeaderWrite::Range,
+      DataFlow::Node
+    {
+      DataFlow::Node name;
+      DataFlow::Node value;
+
+      HeaderWriteSubscript() {
+        exists(SubscriptNode subscript |
+          this.asCfgNode() = subscript and
+          value.asCfgNode() = subscript.(DefinitionNode).getValue() and
+          name.asCfgNode() = subscript.getIndex() and
+          subscript.getObject() = instance().asCfgNode()
+        )
+      }
+
+      override DataFlow::Node getNameArg() { result = name }
+
+      override DataFlow::Node getValueArg() { result = value }
+
+      override predicate nameAllowsNewline() { any() }
+
+      override predicate valueAllowsNewline() { none() }
+    }
+
+    /** A call to `Headers.extend`, assumed to be a response header. */
+    private class HeaderExtendCall extends Http::Server::ResponseHeaderBulkWrite::Range,
+      DataFlow::MethodCallNode
+    {
+      HeaderExtendCall() { this.calls(instance(), "extend") }
+
+      override DataFlow::Node getBulkArg() { result = this.getArg(0) }
+
+      override predicate nameAllowsNewline() { any() }
+
+      override predicate valueAllowsNewline() { none() }
+    }
   }
 
   /**

python/ql/lib/semmle/python/frameworks/data/internal/ApiGraphModels.qll

@@ -168,9 +168,20 @@ module ModelInput {
    * A unit class for adding additional type model rows from CodeQL models.
    */
   class TypeModel extends Unit {
+    /**
+     * Holds if any of the other predicates in this class might have a result
+     * for the given `type`.
+     *
+     * The implementation of this predicate should not depend on `DataFlow::Node`.
+     */
+    bindingset[type]
+    predicate isTypeUsed(string type) { none() }
+
     /**
      * Gets a data-flow node that is a source of the given `type`.
      *
+     * Note that `type` should also be included in `isTypeUsed`.
+     *
      * This must not depend on API graphs, but ensures that an API node is generated for
      * the source.
      */
@@ -180,6 +191,8 @@ module ModelInput {
      * Gets a data-flow node that is a sink of the given `type`,
      * usually because it is an argument passed to a parameter of that type.
      *
+     * Note that `type` should also be included in `isTypeUsed`.
+     *
      * This must not depend on API graphs, but ensures that an API node is generated for
      * the sink.
      */
@@ -188,6 +201,8 @@ module ModelInput {
     /**
      * Gets an API node that is a source or sink of the given `type`.
      *
+     * Note that `type` should also be included in `isTypeUsed`.
+     *
      * Unlike `getASource` and `getASink`, this may depend on API graphs.
      */
     API::Node getAnApiNode(string type) { none() }
@@ -354,6 +369,28 @@ private predicate typeVariableModel(string name, string path) {
   Extensions::typeVariableModel(name, path)
 }
 
+/**
+ * Holds if the given extension tuple `madId` should pretty-print as `model`.
+ *
+ * This predicate should only be used in tests.
+ */
+predicate interpretModelForTest(QlBuiltins::ExtensionId madId, string model) {
+  exists(string type, string path, string kind |
+    Extensions::sourceModel(type, path, kind, madId) and
+    model = "Source: " + type + "; " + path + "; " + kind
+  )
+  or
+  exists(string type, string path, string kind |
+    Extensions::sinkModel(type, path, kind, madId) and
+    model = "Sink: " + type + "; " + path + "; " + kind
+  )
+  or
+  exists(string type, string path, string input, string output, string kind |
+    Extensions::summaryModel(type, path, input, output, kind, madId) and
+    model = "Summary: " + type + "; " + path + "; " + input + "; " + output + "; " + kind
+  )
+}
+
 /**
  * Holds if rows involving `type` might be relevant for the analysis of this database.
  */
@@ -367,6 +404,8 @@ predicate isRelevantType(string type) {
   (
     Specific::isTypeUsed(type)
     or
+    any(TypeModel model).isTypeUsed(type)
+    or
     exists(TestAllModels t)
   )
   or

python/ql/lib/semmle/python/frameworks/data/internal/ApiGraphModelsSpecific.qll

@@ -29,7 +29,11 @@ import semmle.python.dataflow.new.DataFlow::DataFlow as DataFlow
 /**
  * Holds if models describing `type` may be relevant for the analysis of this database.
  */
-predicate isTypeUsed(string type) { API::moduleImportExists(type) }
+bindingset[type]
+predicate isTypeUsed(string type) {
+  // If `type` is a path, then it is the first component that should be imported.
+  API::moduleImportExists(type.splitAt(".", 0))
+}
 
 /**
  * Holds if `type` can be obtained from an instance of `otherType` due to
@@ -41,8 +45,59 @@ predicate hasImplicitTypeModel(string type, string otherType) { none() }
 bindingset[type, path]
 API::Node getExtraNodeFromPath(string type, AccessPath path, int n) { none() }
 
+/**
+ * Holds if `type` = `typePath`+`suffix` and `suffix` is either empty or "!".
+ */
+bindingset[type]
+private predicate parseType(string type, string typePath, string suffix) {
+  exists(string regexp |
+    regexp = "([^!]+)(!|)" and
+    typePath = type.regexpCapture(regexp, 1) and
+    suffix = type.regexpCapture(regexp, 2)
+  )
+}
+
+private predicate parseRelevantType(string type, string typePath, string suffix) {
+  isRelevantType(type) and
+  parseType(type, typePath, suffix)
+}
+
+pragma[nomagic]
+private string getTypePathComponent(string typePath, int n) {
+  parseRelevantType(_, typePath, _) and
+  result = typePath.splitAt(".", n)
+}
+
+private int getNumTypePathComponents(string typePath) {
+  result = strictcount(int n | exists(getTypePathComponent(typePath, n)))
+}
+
+private API::Node getNodeFromTypePath(string typePath, int n) {
+  n = 1 and
+  result = API::moduleImport(getTypePathComponent(typePath, 0))
+  or
+  result = getNodeFromTypePath(typePath, n - 1).getMember(getTypePathComponent(typePath, n - 1))
+}
+
+private API::Node getNodeFromTypePath(string typePath) {
+  result = getNodeFromTypePath(typePath, getNumTypePathComponents(typePath))
+}
+
 /** Gets a Python-specific interpretation of the given `type`. */
-API::Node getExtraNodeFromType(string type) { result = API::moduleImport(type) }
+API::Node getExtraNodeFromType(string type) {
+  result = API::moduleImport(type)
+  or
+  exists(string typePath, string suffix, API::Node node |
+    parseRelevantType(type, typePath, suffix) and
+    node = getNodeFromTypePath(typePath)
+  |
+    suffix = "!" and
+    result = node
+    or
+    suffix = "" and
+    result = node.getAnInstance()
+  )
+}
 
 /**
  * Gets a Python-specific API graph successor of `node` reachable by resolving `token`.
@@ -53,7 +108,7 @@ API::Node getExtraSuccessorFromNode(API::Node node, AccessPathTokenBase token) {
   result = node.getMember(token.getAnArgument())
   or
   token.getName() = "Instance" and
-  result = node.getReturn() // In Python `Instance` is just an alias for `ReturnValue`
+  result = node.getAnInstance()
   or
   token.getName() = "Awaited" and
   result = node.getAwaited()

python/ql/lib/semmle/python/security/dataflow/HttpHeaderInjectionCustomizations.qll

@@ -0,0 +1,113 @@
+/**
+ * Provides default sources, sinks, and sanitizers for detecting
+ * "HTTP Header injection" vulnerabilities, as well as extension
+ * points for adding your own.
+ */
+
+import python
+private import semmle.python.Concepts
+private import semmle.python.dataflow.new.DataFlow
+private import semmle.python.dataflow.new.TaintTracking
+private import semmle.python.dataflow.new.RemoteFlowSources
+
+/**
+ * Provides default sources, sinks, and sanitizers for detecting
+ * "HTTP Header injection" vulnerabilities, as well as extension
+ * points for adding your own.
+ */
+module HttpHeaderInjection {
+  /**
+   * A data flow source for "HTTP Header injection" vulnerabilities.
+   */
+  abstract class Source extends DataFlow::Node { }
+
+  /**
+   * A data flow sink for "HTTP Header injection" vulnerabilities.
+   */
+  abstract class Sink extends DataFlow::Node { }
+
+  /**
+   * A data flow sanitizer for "HTTP Header injection" vulnerabilities.
+   */
+  abstract class Sanitizer extends DataFlow::Node { }
+
+  /**
+   * A source of remote user input, considered as a flow source.
+   */
+  class RemoteFlowSourceAsSource extends Source, RemoteFlowSource { }
+
+  /**
+   * A HTTP header write, considered as a flow sink.
+   */
+  class HeaderWriteAsSink extends Sink {
+    HeaderWriteAsSink() {
+      exists(Http::Server::ResponseHeaderWrite headerWrite |
+        headerWrite.nameAllowsNewline() and
+        this = headerWrite.getNameArg()
+        or
+        headerWrite.valueAllowsNewline() and
+        this = headerWrite.getValueArg()
+      )
+    }
+  }
+
+  /** A key-value pair in a literal for a bulk header update, considered as a single header update. */
+  // TODO: We could instead consider bulk writes as sinks with an implicit read step of DictionaryKey/DictionaryValue content as needed.
+  private class HeaderBulkWriteDictLiteral extends Http::Server::ResponseHeaderWrite::Range instanceof Http::Server::ResponseHeaderBulkWrite
+  {
+    KeyValuePair item;
+
+    HeaderBulkWriteDictLiteral() {
+      exists(Dict dict | DataFlow::localFlow(DataFlow::exprNode(dict), super.getBulkArg()) |
+        item = dict.getAnItem()
+      )
+    }
+
+    override DataFlow::Node getNameArg() { result.asExpr() = item.getKey() }
+
+    override DataFlow::Node getValueArg() { result.asExpr() = item.getValue() }
+
+    override predicate nameAllowsNewline() {
+      Http::Server::ResponseHeaderBulkWrite.super.nameAllowsNewline()
+    }
+
+    override predicate valueAllowsNewline() {
+      Http::Server::ResponseHeaderBulkWrite.super.valueAllowsNewline()
+    }
+  }
+
+  /** A tuple in a list for a bulk header update, considered as a single header update. */
+  // TODO: We could instead consider bulk writes as sinks with implicit read steps as needed.
+  private class HeaderBulkWriteListLiteral extends Http::Server::ResponseHeaderWrite::Range instanceof Http::Server::ResponseHeaderBulkWrite
+  {
+    Tuple item;
+
+    HeaderBulkWriteListLiteral() {
+      exists(List list | DataFlow::localFlow(DataFlow::exprNode(list), super.getBulkArg()) |
+        item = list.getAnElt()
+      )
+    }
+
+    override DataFlow::Node getNameArg() { result.asExpr() = item.getElt(0) }
+
+    override DataFlow::Node getValueArg() { result.asExpr() = item.getElt(1) }
+
+    override predicate nameAllowsNewline() {
+      Http::Server::ResponseHeaderBulkWrite.super.nameAllowsNewline()
+    }
+
+    override predicate valueAllowsNewline() {
+      Http::Server::ResponseHeaderBulkWrite.super.valueAllowsNewline()
+    }
+  }
+
+  /**
+   * A call to replace line breaks, considered as a sanitizer.
+   */
+  class ReplaceLineBreaksSanitizer extends Sanitizer, DataFlow::CallCfgNode {
+    ReplaceLineBreaksSanitizer() {
+      this.getFunction().(DataFlow::AttrRead).getAttributeName() = "replace" and
+      this.getArg(0).asExpr().(StringLiteral).getText() = "\n"
+    }
+  }
+}

python/ql/lib/semmle/python/security/dataflow/HttpHeaderInjectionQuery.qll

@@ -0,0 +1,22 @@
+/**
+ * Provides a taint tracking configuration for reasoning about HTTP header injection.
+ */
+
+import python
+private import semmle.python.dataflow.new.DataFlow
+private import semmle.python.dataflow.new.TaintTracking
+private import HttpHeaderInjectionCustomizations
+
+/**
+ * A taint-tracking configuration for detecting HTTP Header injection vulnerabilities.
+ */
+private module HeaderInjectionConfig implements DataFlow::ConfigSig {
+  predicate isSource(DataFlow::Node node) { node instanceof HttpHeaderInjection::Source }
+
+  predicate isSink(DataFlow::Node node) { node instanceof HttpHeaderInjection::Sink }
+
+  predicate isBarrier(DataFlow::Node node) { node instanceof HttpHeaderInjection::Sanitizer }
+}
+
+/** Global taint-tracking for detecting "HTTP Header injection" vulnerabilities. */
+module HeaderInjectionFlow = TaintTracking::Global<HeaderInjectionConfig>;

python/ql/lib/semmle/python/security/dataflow/ServerSideRequestForgeryCustomizations.qll

@@ -9,6 +9,7 @@ private import semmle.python.dataflow.new.DataFlow
 private import semmle.python.Concepts
 private import semmle.python.dataflow.new.RemoteFlowSources
 private import semmle.python.dataflow.new.BarrierGuards
+private import semmle.python.ApiGraphs
 
 /**
  * Provides default sources, sinks and sanitizers for detecting
@@ -137,4 +138,34 @@ module ServerSideRequestForgery {
       )
     }
   }
+
+  /** A validation that a string does not contain certain characters, considered as a sanitizer. */
+  private class StringRestrictionSanitizerGuard extends Sanitizer {
+    StringRestrictionSanitizerGuard() {
+      this = DataFlow::BarrierGuard<stringRestriction/3>::getABarrierNode()
+    }
+  }
+
+  private predicate stringRestriction(DataFlow::GuardNode g, ControlFlowNode node, boolean branch) {
+    exists(DataFlow::MethodCallNode call, DataFlow::Node strNode |
+      call.asCfgNode() = g and strNode.asCfgNode() = node
+    |
+      branch = true and
+      call.calls(strNode,
+        ["isalnum", "isalpha", "isdecimal", "isdigit", "isidentifier", "isnumeric", "isspace"])
+      or
+      branch = true and
+      call = API::moduleImport("re").getMember(["match", "fullmatch"]).getACall() and
+      strNode = [call.getArg(1), call.getArgByName("string")]
+      or
+      branch = true and
+      call =
+        API::moduleImport("re")
+            .getMember("compile")
+            .getReturn()
+            .getMember(["match", "fullmatch"])
+            .getACall() and
+      strNode = [call.getArg(0), call.getArgByName("string")]
+    )
+  }
 }

python/ql/lib/semmle/python/security/dataflow/ServerSideRequestForgeryQuery.qll

@@ -74,7 +74,7 @@ predicate fullyControlledRequest(Http::Client::Request request) {
 }
 
 /**
- * DEPRECATED: Use `FullServerSideRequestForgeryFlow` module instead.
+ * DEPRECATED: Use `PartialServerSideRequestForgeryFlow` module instead.
  *
  * A taint-tracking configuration for detecting "Server-side request forgery" vulnerabilities.
  *

python/ql/lib/semmle/python/security/internal/SensitiveDataHeuristics.qll

@@ -14,13 +14,14 @@
  *   - id: a user name or other account information;
  *   - password: a password or authorization key;
  *   - certificate: a certificate.
+ *   - private: private data such as credit card numbers
  *
  * While classifications are represented as strings, this should not be relied upon.
  * Instead, use the predicates in `SensitiveDataClassification::` to work with
  * classifications.
  */
 class SensitiveDataClassification extends string {
-  SensitiveDataClassification() { this in ["secret", "id", "password", "certificate"] }
+  SensitiveDataClassification() { this in ["secret", "id", "password", "certificate", "private"] }
 }
 
 /**
@@ -38,6 +39,9 @@ module SensitiveDataClassification {
 
   /** Gets the classification for certificates. */
   SensitiveDataClassification certificate() { result = "certificate" }
+
+  /** Gets the classification for private data. */
+  SensitiveDataClassification private() { result = "private" }
 }
 
 /**
@@ -77,6 +81,40 @@ module HeuristicNames {
    */
   string maybeCertificate() { result = "(?is).*(cert)(?!.*(format|name|ification)).*" }
 
+  /**
+   * Gets a regular expression that identifies strings that may indicate the presence of
+   * private data.
+   */
+  string maybePrivate() {
+    result =
+      "(?is).*(" +
+        // Inspired by the list on https://cwe.mitre.org/data/definitions/359.html
+        // Government identifiers, such as Social Security Numbers
+        "social.?security|employer.?identification|national.?insurance|resident.?id|" +
+        "passport.?(num|no)|([_-]|\\b)ssn([_-]|\\b)|" +
+        // Contact information, such as home addresses
+        "post.?code|zip.?code|home.?addr|" +
+        // and telephone numbers
+        "(mob(ile)?|home).?(num|no|tel|phone)|(tel|fax|phone).?(num|no)|telephone|" +
+        "emergency.?contact|" +
+        // Geographic location - where the user is (or was)
+        "latitude|longitude|nationality|" +
+        // Financial data - such as credit card numbers, salary, bank accounts, and debts
+        "(credit|debit|bank|visa).?(card|num|no|acc(ou)?nt)|acc(ou)?nt.?(no|num|credit)|" +
+        "salary|billing|credit.?(rating|score)|([_-]|\\b)ccn([_-]|\\b)|" +
+        // Communications - e-mail addresses, private e-mail messages, SMS text messages, chat logs, etc.
+        // "e(mail|_mail)|" + // this seems too noisy
+        // Health - medical conditions, insurance status, prescription records
+        "birth.?da(te|y)|da(te|y).?(of.?)?birth|" +
+        "medical|(health|care).?plan|healthkit|appointment|prescription|" +
+        "blood.?(type|alcohol|glucose|pressure)|heart.?(rate|rhythm)|body.?(mass|fat)|" +
+        "menstrua|pregnan|insulin|inhaler|" +
+        // Relationships - work and family
+        "employ(er|ee)|spouse|maiden.?name" +
+        // ---
+        ").*"
+  }
+
   /**
    * Gets a regular expression that identifies strings that may indicate the presence
    * of sensitive data, with `classification` describing the kind of sensitive data involved.
@@ -90,6 +128,9 @@ module HeuristicNames {
     or
     result = maybeCertificate() and
     classification = SensitiveDataClassification::certificate()
+    or
+    result = maybePrivate() and
+    classification = SensitiveDataClassification::private()
   }
 
   /**

python/ql/lib/upgrades/4f1806347d7fafe2f78508da01c01e5aff5f7cbb/upgrade.properties

@@ -1,4 +1,4 @@
 description: Removed unused column from the `folders` and `files` relations
 compatibility: full
-files.rel: reorder files.rel (int id, string name, string simple, string ext, int fromSource) id name
-folders.rel: reorder folders.rel (int id, string name, string simple) id name
+files.rel: reorder files.rel (@file id, string name, string simple, string ext, int fromSource) id name
+folders.rel: reorder folders.rel (@folder id, string name, string simple) id name