Merge branch 'main' of github.com:github/codeql into python-dataflow/flow-summaries-from-scratch

python/ql/lib/CHANGELOG.md

@@ -1,3 +1,16 @@
+## 0.5.4
+
+### Deprecated APIs
+
+* Many classes/predicates/modules with upper-case acronyms in their name have been renamed to follow our style-guide. 
+  The old name still exists as a deprecated alias.
+* The utility files previously in the `semmle.python.security.performance` package have been moved to the `semmle.python.security.regexp` package.  
+  The previous files still exist as deprecated aliases.
+
+### Minor Analysis Improvements
+
+* Most deprecated predicates/classes/modules that have been deprecated for over a year have been deleted.
+
 ## 0.5.3
 
 ### Minor Analysis Improvements

python/ql/lib/change-notes/2022-05-25-redos-refac.md

@@ -1,5 +0,0 @@
----
-category: deprecated
----
-* The utility files previously in the `semmle.python.security.performance` package have been moved to the `semmle.python.security.regexp` package.  
-  The previous files still exist as deprecated aliases.

python/ql/lib/change-notes/2022-08-17-deleted-deprecations.md

@@ -1,6 +0,0 @@
----
-category: minorAnalysis
----
-* Most deprecated predicates/classes/modules that have been deprecated for over a year have been 
-deleted.
-

python/ql/lib/change-notes/2022-08-22-xml-rename.md

@@ -1,5 +0,0 @@
----
-category: deprecated
----
-* Many classes/predicates/modules with upper-case acronyms in their name have been renamed to follow our style-guide. 
-  The old name still exists as a deprecated alias.

python/ql/lib/change-notes/2022-08-25-variable-fix.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* Reads of global/non-local variables (without annotations) inside functions defined on classes now works properly in the case where the class had an attribute defined with the same name as the non-local variable.

python/ql/lib/change-notes/2022-09-08-implicit-read-flowstates.md

@@ -0,0 +1,4 @@
+---
+category: fix
+---
+* Fixed an issue in the taint tracking analysis where implicit reads were not allowed by default in sinks or additional taint steps that used flow states.

python/ql/lib/change-notes/2022-09-12-deprecate-pointsto.md

@@ -0,0 +1,4 @@
+---
+category: deprecated
+---
+* Some unused predicates in `SsaDefinitions.qll`, `TObject.qll`, `protocols.qll`, and the `pointsto/` folder have been deprecated.

python/ql/lib/change-notes/released/0.5.4.md

@@ -0,0 +1,12 @@
+## 0.5.4
+
+### Deprecated APIs
+
+* Many classes/predicates/modules with upper-case acronyms in their name have been renamed to follow our style-guide. 
+  The old name still exists as a deprecated alias.
+* The utility files previously in the `semmle.python.security.performance` package have been moved to the `semmle.python.security.regexp` package.  
+  The previous files still exist as deprecated aliases.
+
+### Minor Analysis Improvements
+
+* Most deprecated predicates/classes/modules that have been deprecated for over a year have been deleted.

python/ql/lib/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.5.3
+lastReleaseVersion: 0.5.4

python/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/python-all
-version: 0.5.4-dev
+version: 0.5.5-dev
 groups: python
 dbscheme: semmlecode.python.dbscheme
 extractor: python

python/ql/lib/semmle/python/ApiGraphs.qll

@@ -468,6 +468,30 @@ module API {
     int getNumArgument() { result = count(this.getArg(_)) }
   }
 
+  /**
+   * An API entry point.
+   *
+   * By default, API graph nodes are only created for nodes that come from an external
+   * library or escape into an external library. The points where values are cross the boundary
+   * between codebases are called "entry points".
+   *
+   * Anything imported from an external package is considered to be an entry point, but
+   * additional entry points may be added by extending this class.
+   */
+  abstract class EntryPoint extends string {
+    bindingset[this]
+    EntryPoint() { any() }
+
+    /** Gets a data-flow node corresponding to a use-node for this entry point. */
+    DataFlow::LocalSourceNode getASource() { none() }
+
+    /** Gets a data-flow node corresponding to a def-node for this entry point. */
+    DataFlow::Node getASink() { none() }
+
+    /** Gets an API-node for this entry point. */
+    API::Node getANode() { result = root().getASuccessor(Label::entryPoint(this)) }
+  }
+
   /**
    * Provides the actual implementation of API graphs, cached for performance.
    *
@@ -691,6 +715,12 @@ module API {
       |
         lbl = Label::memberFromRef(aw)
       )
+      or
+      exists(EntryPoint entry |
+        base = root() and
+        lbl = Label::entryPoint(entry) and
+        rhs = entry.getASink()
+      )
     }
 
     /**
@@ -775,6 +805,12 @@ module API {
               ImportStar::namePossiblyDefinedInImportStar(ref.(DataFlow::CfgNode).getNode(), name, s)
             ))
       )
+      or
+      exists(EntryPoint entry |
+        base = root() and
+        lbl = Label::entryPoint(entry) and
+        ref = entry.getASource()
+      )
     }
 
     /**
@@ -949,7 +985,8 @@ module API {
         MkLabelSelfParameter() or
         MkLabelReturn() or
         MkLabelSubclass() or
-        MkLabelAwait()
+        MkLabelAwait() or
+        MkLabelEntryPoint(EntryPoint ep)
 
       /** A label for a module. */
       class LabelModule extends ApiLabel, MkLabelModule {
@@ -1023,6 +1060,15 @@ module API {
       class LabelAwait extends ApiLabel, MkLabelAwait {
         override string toString() { result = "getAwaited()" }
       }
+
+      /** A label for entry points. */
+      class LabelEntryPoint extends ApiLabel, MkLabelEntryPoint {
+        private EntryPoint entry;
+
+        LabelEntryPoint() { this = MkLabelEntryPoint(entry) }
+
+        override string toString() { result = "entryPoint(\"" + entry + "\")" }
+      }
     }
 
     /** Gets the edge label for the module `m`. */
@@ -1059,5 +1105,8 @@ module API {
 
     /** Gets the `await` edge label. */
     LabelAwait await() { any() }
+
+    /** Gets the label going from the root node to the nodes associated with the given entry point. */
+    LabelEntryPoint entryPoint(EntryPoint ep) { result = MkLabelEntryPoint(ep) }
   }
 }

python/ql/lib/semmle/python/Concepts.qll

@@ -1046,71 +1046,9 @@ module HTTP {
     }
   }
 
-  /** Provides classes for modeling HTTP clients. */
-  module Client {
-    /**
-     * A data-flow node that makes an outgoing HTTP request.
-     *
-     * Extend this class to refine existing API models. If you want to model new APIs,
-     * extend `HTTP::Client::Request::Range` instead.
-     */
-    class Request extends DataFlow::Node instanceof Request::Range {
-      /**
-       * Gets a data-flow node that contributes to the URL of the request.
-       * Depending on the framework, a request may have multiple nodes which contribute to the URL.
-       */
-      DataFlow::Node getAUrlPart() { result = super.getAUrlPart() }
-
-      /** Gets a string that identifies the framework used for this request. */
-      string getFramework() { result = super.getFramework() }
-
-      /**
-       * Holds if this request is made using a mode that disables SSL/TLS
-       * certificate validation, where `disablingNode` represents the point at
-       * which the validation was disabled, and `argumentOrigin` represents the origin
-       * of the argument that disabled the validation (which could be the same node as
-       * `disablingNode`).
-       */
-      predicate disablesCertificateValidation(
-        DataFlow::Node disablingNode, DataFlow::Node argumentOrigin
-      ) {
-        super.disablesCertificateValidation(disablingNode, argumentOrigin)
-      }
-    }
-
-    /** Provides a class for modeling new HTTP requests. */
-    module Request {
-      /**
-       * A data-flow node that makes an outgoing HTTP request.
-       *
-       * Extend this class to model new APIs. If you want to refine existing API models,
-       * extend `HTTP::Client::Request` instead.
-       */
-      abstract class Range extends DataFlow::Node {
-        /**
-         * Gets a data-flow node that contributes to the URL of the request.
-         * Depending on the framework, a request may have multiple nodes which contribute to the URL.
-         */
-        abstract DataFlow::Node getAUrlPart();
-
-        /** Gets a string that identifies the framework used for this request. */
-        abstract string getFramework();
-
-        /**
-         * Holds if this request is made using a mode that disables SSL/TLS
-         * certificate validation, where `disablingNode` represents the point at
-         * which the validation was disabled, and `argumentOrigin` represents the origin
-         * of the argument that disabled the validation (which could be the same node as
-         * `disablingNode`).
-         */
-        abstract predicate disablesCertificateValidation(
-          DataFlow::Node disablingNode, DataFlow::Node argumentOrigin
-        );
-      }
-    }
-    // TODO: investigate whether we should treat responses to client requests as
-    // remote-flow-sources in general.
-  }
+  import semmle.python.internal.ConceptsShared::Http::Client as Client
+  // TODO: investigate whether we should treat responses to client requests as
+  // remote-flow-sources in general.
 }
 
 /**

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowPrivate.qll

@@ -1,7 +1,7 @@
 private import python
 private import DataFlowPublic
 private import semmle.python.essa.SsaCompute
-private import semmle.python.dataflow.new.internal.ImportStar
+private import semmle.python.dataflow.new.internal.ImportResolution
 private import FlowSummaryImpl as FlowSummaryImpl
 // Since we allow extra data-flow steps from modeled frameworks, we import these
 // up-front, to ensure these are included. This provides a more seamless experience from
@@ -387,11 +387,7 @@ predicate runtimeJumpStep(Node nodeFrom, Node nodeTo) {
   nodeFrom = nodeTo.(ModuleVariableNode).getAWrite()
   or
   // Setting the possible values of the variable at the end of import time
-  exists(SsaVariable def |
-    def = any(SsaVariable var).getAnUltimateDefinition() and
-    def.getDefinition() = nodeFrom.asCfgNode() and
-    def.getVariable() = nodeTo.(ModuleVariableNode).getVariable()
-  )
+  nodeFrom = nodeTo.(ModuleVariableNode).getADefiningWrite()
 }
 
 /**
@@ -475,9 +471,9 @@ predicate jumpStepSharedWithTypeTracker(Node nodeFrom, Node nodeTo) {
   runtimeJumpStep(nodeFrom, nodeTo)
   or
   // Read of module attribute:
-  exists(AttrRead r, ModuleValue mv |
-    r.getObject().asCfgNode().pointsTo(mv) and
-    module_export(mv.getScope(), r.getAttributeName(), nodeFrom) and
+  exists(AttrRead r |
+    ImportResolution::module_export(ImportResolution::getModule(r.getObject()),
+      r.getAttributeName(), nodeFrom) and
     nodeTo = r
   )
   or
@@ -501,22 +497,6 @@ predicate jumpStepNotSharedWithTypeTracker(Node nodeFrom, Node nodeTo) {
   any(Orm::AdditionalOrmSteps es).jumpStep(nodeFrom, nodeTo)
 }
 
-/**
- * Holds if the module `m` defines a name `name` by assigning `defn` to it. This is an
- * overapproximation, as `name` may not in fact be exported (e.g. by defining an `__all__` that does
- * not include `name`).
- */
-private predicate module_export(Module m, string name, CfgNode defn) {
-  exists(EssaVariable v |
-    v.getName() = name and
-    v.getAUse() = ImportStar::getStarImported*(m).getANormalExit()
-  |
-    defn.getNode() = v.getDefinition().(AssignmentDefinition).getValue()
-    or
-    defn.getNode() = v.getDefinition().(ArgumentRefinement).getArgument()
-  )
-}
-
 //--------
 // Field flow
 //--------

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowPublic.qll

@@ -420,6 +420,15 @@ class ModuleVariableNode extends Node, TModuleVariableNode {
     result.getVar().getDefinition().(EssaNodeDefinition).definedBy(var, any(DefinitionNode defn))
   }
 
+  /** Gets the possible values of the variable at the end of import time */
+  CfgNode getADefiningWrite() {
+    exists(SsaVariable def |
+      def = any(SsaVariable ssa_var).getAnUltimateDefinition() and
+      def.getDefinition() = result.asCfgNode() and
+      def.getVariable() = var
+    )
+  }
+
   override DataFlowCallable getEnclosingCallable() { result.(DataFlowModuleScope).getScope() = mod }
 
   override Location getLocation() { result = mod.getLocation() }

python/ql/lib/semmle/python/dataflow/new/internal/ImportResolution.qll

@@ -0,0 +1,29 @@
+private import python
+private import semmle.python.dataflow.new.DataFlow
+private import semmle.python.dataflow.new.internal.ImportStar
+private import semmle.python.dataflow.new.TypeTracker
+
+module ImportResolution {
+  /**
+   * Holds if the module `m` defines a name `name` by assigning `defn` to it. This is an
+   * overapproximation, as `name` may not in fact be exported (e.g. by defining an `__all__` that does
+   * not include `name`).
+   */
+  predicate module_export(Module m, string name, DataFlow::CfgNode defn) {
+    exists(EssaVariable v |
+      v.getName() = name and
+      v.getAUse() = ImportStar::getStarImported*(m).getANormalExit()
+    |
+      defn.getNode() = v.getDefinition().(AssignmentDefinition).getValue()
+      or
+      defn.getNode() = v.getDefinition().(ArgumentRefinement).getArgument()
+    )
+  }
+
+  Module getModule(DataFlow::CfgNode node) {
+    exists(ModuleValue mv |
+      node.getNode().pointsTo(mv) and
+      result = mv.getScope()
+    )
+  }
+}

python/ql/lib/semmle/python/dataflow/new/internal/ImportStar.qll

@@ -2,6 +2,8 @@
 
 private import python
 private import semmle.python.dataflow.new.internal.Builtins
+private import semmle.python.dataflow.new.internal.ImportResolution
+private import semmle.python.dataflow.new.DataFlow
 
 cached
 module ImportStar {
@@ -71,8 +73,10 @@ module ImportStar {
    */
   cached
   Module getStarImported(Module m) {
-    exists(ImportStar i |
-      i.getScope() = m and result = i.getModule().pointsTo().(ModuleValue).getScope()
+    exists(ImportStar i, DataFlow::CfgNode imported_module |
+      imported_module.getNode().getNode() = i.getModule() and
+      i.getScope() = m and
+      result = ImportResolution::getModule(imported_module)
     )
   }
 

python/ql/lib/semmle/python/dataflow/new/internal/tainttracking1/TaintTrackingImpl.qll

@@ -172,7 +172,12 @@ abstract class Configuration extends DataFlow::Configuration {
   }
 
   override predicate allowImplicitRead(DataFlow::Node node, DataFlow::ContentSet c) {
-    (this.isSink(node) or this.isAdditionalTaintStep(node, _)) and
+    (
+      this.isSink(node) or
+      this.isSink(node, _) or
+      this.isAdditionalTaintStep(node, _) or
+      this.isAdditionalTaintStep(node, _, _, _)
+    ) and
     defaultImplicitTaintRead(node, c)
   }
 

python/ql/lib/semmle/python/dataflow/new/internal/tainttracking2/TaintTrackingImpl.qll

@@ -172,7 +172,12 @@ abstract class Configuration extends DataFlow::Configuration {
   }
 
   override predicate allowImplicitRead(DataFlow::Node node, DataFlow::ContentSet c) {
-    (this.isSink(node) or this.isAdditionalTaintStep(node, _)) and
+    (
+      this.isSink(node) or
+      this.isSink(node, _) or
+      this.isAdditionalTaintStep(node, _) or
+      this.isAdditionalTaintStep(node, _, _, _)
+    ) and
     defaultImplicitTaintRead(node, c)
   }
 

python/ql/lib/semmle/python/dataflow/new/internal/tainttracking3/TaintTrackingImpl.qll

@@ -172,7 +172,12 @@ abstract class Configuration extends DataFlow::Configuration {
   }
 
   override predicate allowImplicitRead(DataFlow::Node node, DataFlow::ContentSet c) {
-    (this.isSink(node) or this.isAdditionalTaintStep(node, _)) and
+    (
+      this.isSink(node) or
+      this.isSink(node, _) or
+      this.isAdditionalTaintStep(node, _) or
+      this.isAdditionalTaintStep(node, _, _, _)
+    ) and
     defaultImplicitTaintRead(node, c)
   }
 

python/ql/lib/semmle/python/dataflow/new/internal/tainttracking4/TaintTrackingImpl.qll

@@ -172,7 +172,12 @@ abstract class Configuration extends DataFlow::Configuration {
   }
 
   override predicate allowImplicitRead(DataFlow::Node node, DataFlow::ContentSet c) {
-    (this.isSink(node) or this.isAdditionalTaintStep(node, _)) and
+    (
+      this.isSink(node) or
+      this.isSink(node, _) or
+      this.isAdditionalTaintStep(node, _) or
+      this.isAdditionalTaintStep(node, _, _, _)
+    ) and
     defaultImplicitTaintRead(node, c)
   }
 

python/ql/lib/semmle/python/essa/SsaDefinitions.qll

@@ -78,7 +78,9 @@ module SsaSource {
 
   /** Holds if `v` is defined by a `for` statement, the definition being `defn` */
   cached
-  predicate iteration_defined_variable(Variable v, ControlFlowNode defn, ControlFlowNode sequence) {
+  deprecated predicate iteration_defined_variable(
+    Variable v, ControlFlowNode defn, ControlFlowNode sequence
+  ) {
     exists(ForNode for | for.iterates(defn, sequence)) and
     defn.(NameNode).defines(v)
   }

python/ql/lib/semmle/python/frameworks/Stdlib.qll

@@ -2098,8 +2098,8 @@ private module StdlibPrivate {
      *
      * See https://docs.python.org/3.10/library/wsgiref.html#wsgiref.simple_server.WSGIRequestHandler.get_environ
      */
-    class WSGIEnvirontParameter extends RemoteFlowSource::Range, DataFlow::ParameterNode {
-      WSGIEnvirontParameter() {
+    class WsgiEnvirontParameter extends RemoteFlowSource::Range, DataFlow::ParameterNode {
+      WsgiEnvirontParameter() {
         exists(WsgirefSimpleServerApplication func |
           if func.isMethod()
           then this.getParameter() = func.getArg(1)
@@ -2112,6 +2112,9 @@ private module StdlibPrivate {
       }
     }
 
+    /** DEPRECATED: Alias for WsgiEnvirontParameter */
+    deprecated class WSGIEnvirontParameter = WsgiEnvirontParameter;
+
     /**
      * Gets a reference to the parameter of a `WsgirefSimpleServerApplication` that
      * takes the `start_response` function.

python/ql/lib/semmle/python/frameworks/data/internal/ApiGraphModels.qll

@@ -72,6 +72,8 @@ private class Unit = Specific::Unit;
 
 private module API = Specific::API;
 
+private module DataFlow = Specific::DataFlow;
+
 private import Specific::AccessPathSyntax
 
 /** Module containing hooks for providing input data to be interpreted as a model. */
@@ -155,6 +157,38 @@ module ModelInput {
      */
     abstract predicate row(string row);
   }
+
+  /**
+   * A unit class for adding additional type model rows from CodeQL models.
+   */
+  class TypeModel extends Unit {
+    /**
+     * Gets a data-flow node that is a source of the type `package;type`.
+     */
+    DataFlow::Node getASource(string package, string type) { none() }
+
+    /**
+     * Gets a data flow node that is a sink of the type `package;type`,
+     * usually because it is an argument passed to a parameter of that type.
+     */
+    DataFlow::Node getASink(string package, string type) { none() }
+  }
+
+  /**
+   * A unit class for adding additional type variable model rows.
+   */
+  class TypeVariableModelCsv extends Unit {
+    /**
+     * Holds if `row` specifies a path through a type variable.
+     *
+     * A row of form,
+     * ```
+     * name;path
+     * ```
+     * means `path` can be substituted for a token `TypeVar[name]`.
+     */
+    abstract predicate row(string row);
+  }
 }
 
 private import ModelInput
@@ -182,6 +216,8 @@ private predicate summaryModel(string row) { any(SummaryModelCsv s).row(inverseP
 
 private predicate typeModel(string row) { any(TypeModelCsv s).row(inversePad(row)) }
 
+private predicate typeVariableModel(string row) { any(TypeVariableModelCsv s).row(inversePad(row)) }
+
 /** Holds if a source model exists for the given parameters. */
 predicate sourceModel(string package, string type, string path, string kind) {
   exists(string row |
@@ -219,7 +255,7 @@ private predicate summaryModel(
   )
 }
 
-/** Holds if an type model exists for the given parameters. */
+/** Holds if a type model exists for the given parameters. */
 private predicate typeModel(
   string package1, string type1, string package2, string type2, string path
 ) {
@@ -233,6 +269,15 @@ private predicate typeModel(
   )
 }
 
+/** Holds if a type variable model exists for the given parameters. */
+private predicate typeVariableModel(string name, string path) {
+  exists(string row |
+    typeVariableModel(row) and
+    row.splitAt(";", 0) = name and
+    row.splitAt(";", 1) = path
+  )
+}
+
 /**
  * Gets a package that should be seen as an alias for the given other `package`,
  * or the `package` itself.
@@ -253,7 +298,7 @@ private predicate isRelevantPackage(string package) {
     sourceModel(package, _, _, _) or
     sinkModel(package, _, _, _) or
     summaryModel(package, _, _, _, _, _) or
-    typeModel(package, _, _, _, _)
+    typeModel(_, _, package, _, _)
   ) and
   (
     Specific::isPackageUsed(package)
@@ -290,6 +335,8 @@ private class AccessPathRange extends AccessPath::Range {
       summaryModel(package, _, _, this, _, _) or
       summaryModel(package, _, _, _, this, _)
     )
+    or
+    typeVariableModel(_, this)
   }
 }
 
@@ -339,6 +386,57 @@ private predicate invocationMatchesCallSiteFilter(Specific::InvokeNode invoke, A
   Specific::invocationMatchesExtraCallSiteFilter(invoke, token)
 }
 
+private class TypeModelUseEntry extends API::EntryPoint {
+  private string package;
+  private string type;
+
+  TypeModelUseEntry() {
+    exists(any(TypeModel tm).getASource(package, type)) and
+    this = "TypeModelUseEntry;" + package + ";" + type
+  }
+
+  override DataFlow::LocalSourceNode getASource() {
+    result = any(TypeModel tm).getASource(package, type)
+  }
+
+  API::Node getNodeForType(string package_, string type_) {
+    package = package_ and type = type_ and result = this.getANode()
+  }
+}
+
+private class TypeModelDefEntry extends API::EntryPoint {
+  private string package;
+  private string type;
+
+  TypeModelDefEntry() {
+    exists(any(TypeModel tm).getASink(package, type)) and
+    this = "TypeModelDefEntry;" + package + ";" + type
+  }
+
+  override DataFlow::Node getASink() { result = any(TypeModel tm).getASink(package, type) }
+
+  API::Node getNodeForType(string package_, string type_) {
+    package = package_ and type = type_ and result = this.getANode()
+  }
+}
+
+/**
+ * Gets an API node identified by the given `(package,type)` pair.
+ */
+pragma[nomagic]
+private API::Node getNodeFromType(string package, string type) {
+  exists(string package2, string type2, AccessPath path2 |
+    typeModel(package, type, package2, type2, path2) and
+    result = getNodeFromPath(package2, type2, path2)
+  )
+  or
+  result = any(TypeModelUseEntry e).getNodeForType(package, type)
+  or
+  result = any(TypeModelDefEntry e).getNodeForType(package, type)
+  or
+  result = Specific::getExtraNodeFromType(package, type)
+}
+
 /**
  * Gets the API node identified by the first `n` tokens of `path` in the given `(package, type, path)` tuple.
  */
@@ -347,12 +445,8 @@ private API::Node getNodeFromPath(string package, string type, AccessPath path,
   isRelevantFullPath(package, type, path) and
   (
     n = 0 and
-    exists(string package2, string type2, AccessPath path2 |
-      typeModel(package, type, package2, type2, path2) and
-      result = getNodeFromPath(package2, type2, path2, path2.getNumToken())
-    )
+    result = getNodeFromType(package, type)
     or
-    // Language-specific cases, such as handling of global variables
     result = Specific::getExtraNodeFromPath(package, type, path, n)
   )
   or
@@ -361,6 +455,72 @@ private API::Node getNodeFromPath(string package, string type, AccessPath path,
   // Similar to the other recursive case, but where the path may have stepped through one or more call-site filters
   result =
     getSuccessorFromInvoke(getInvocationFromPath(package, type, path, n - 1), path.getToken(n - 1))
+  or
+  // Apply a subpath
+  result =
+    getNodeFromSubPath(getNodeFromPath(package, type, path, n - 1), getSubPathAt(path, n - 1))
+  or
+  // Apply a type step
+  typeStep(getNodeFromPath(package, type, path, n), result)
+}
+
+/**
+ * Gets a subpath for the `TypeVar` token found at the `n`th token of `path`.
+ */
+pragma[nomagic]
+private AccessPath getSubPathAt(AccessPath path, int n) {
+  exists(string typeVarName |
+    path.getToken(n).getAnArgument("TypeVar") = typeVarName and
+    typeVariableModel(typeVarName, result)
+  )
+}
+
+/**
+ * Gets a node that is found by evaluating the first `n` tokens of `subPath` starting at `base`.
+ */
+pragma[nomagic]
+private API::Node getNodeFromSubPath(API::Node base, AccessPath subPath, int n) {
+  exists(AccessPath path, int k |
+    base = [getNodeFromPath(_, _, path, k), getNodeFromSubPath(_, path, k)] and
+    subPath = getSubPathAt(path, k) and
+    result = base and
+    n = 0
+  )
+  or
+  exists(string package, string type, AccessPath basePath |
+    typeStepModel(package, type, basePath, subPath) and
+    base = getNodeFromPath(package, type, basePath) and
+    result = base and
+    n = 0
+  )
+  or
+  result = getSuccessorFromNode(getNodeFromSubPath(base, subPath, n - 1), subPath.getToken(n - 1))
+  or
+  result =
+    getSuccessorFromInvoke(getInvocationFromSubPath(base, subPath, n - 1), subPath.getToken(n - 1))
+  or
+  result =
+    getNodeFromSubPath(getNodeFromSubPath(base, subPath, n - 1), getSubPathAt(subPath, n - 1))
+  or
+  typeStep(getNodeFromSubPath(base, subPath, n), result)
+}
+
+/**
+ * Gets a call site that is found by evaluating the first `n` tokens of `subPath` starting at `base`.
+ */
+private Specific::InvokeNode getInvocationFromSubPath(API::Node base, AccessPath subPath, int n) {
+  result = Specific::getAnInvocationOf(getNodeFromSubPath(base, subPath, n))
+  or
+  result = getInvocationFromSubPath(base, subPath, n - 1) and
+  invocationMatchesCallSiteFilter(result, subPath.getToken(n - 1))
+}
+
+/**
+ * Gets a node that is found by evaluating `subPath` starting at `base`.
+ */
+pragma[nomagic]
+private API::Node getNodeFromSubPath(API::Node base, AccessPath subPath) {
+  result = getNodeFromSubPath(base, subPath, subPath.getNumToken())
 }
 
 /** Gets the node identified by the given `(package, type, path)` tuple. */
@@ -368,6 +528,20 @@ API::Node getNodeFromPath(string package, string type, AccessPath path) {
   result = getNodeFromPath(package, type, path, path.getNumToken())
 }
 
+pragma[nomagic]
+private predicate typeStepModel(string package, string type, AccessPath basePath, AccessPath output) {
+  summaryModel(package, type, basePath, "", output, "type")
+}
+
+pragma[nomagic]
+private predicate typeStep(API::Node pred, API::Node succ) {
+  exists(string package, string type, AccessPath basePath, AccessPath output |
+    typeStepModel(package, type, basePath, output) and
+    pred = getNodeFromPath(package, type, basePath) and
+    succ = getNodeFromSubPath(pred, output)
+  )
+}
+
 /**
  * Gets an invocation identified by the given `(package, type, path)` tuple.
  *
@@ -390,7 +564,7 @@ Specific::InvokeNode getInvocationFromPath(string package, string type, AccessPa
  */
 bindingset[name]
 predicate isValidTokenNameInIdentifyingAccessPath(string name) {
-  name = ["Argument", "Parameter", "ReturnValue", "WithArity"]
+  name = ["Argument", "Parameter", "ReturnValue", "WithArity", "TypeVar"]
   or
   Specific::isExtraValidTokenNameInIdentifyingAccessPath(name)
 }
@@ -418,6 +592,9 @@ predicate isValidTokenArgumentInIdentifyingAccessPath(string name, string argume
   name = "WithArity" and
   argument.regexpMatch("\\d+(\\.\\.(\\d+)?)?")
   or
+  name = "TypeVar" and
+  exists(argument)
+  or
   Specific::isExtraValidTokenArgumentInIdentifyingAccessPath(name, argument)
 }
 
@@ -469,12 +646,7 @@ module ModelOutput {
    * Holds if `node` is seen as an instance of `(package,type)` due to a type definition
    * contributed by a CSV model.
    */
-  API::Node getATypeNode(string package, string type) {
-    exists(string package2, string type2, AccessPath path |
-      typeModel(package, type, package2, type2, path) and
-      result = getNodeFromPath(package2, type2, path)
-    )
-  }
+  API::Node getATypeNode(string package, string type) { result = getNodeFromType(package, type) }
 
   /**
    * Gets an error message relating to an invalid CSV row in a model.
@@ -489,6 +661,8 @@ module ModelOutput {
       any(SummaryModelCsv csv).row(row) and kind = "summary" and expectedArity = 6
       or
       any(TypeModelCsv csv).row(row) and kind = "type" and expectedArity = 5
+      or
+      any(TypeVariableModelCsv csv).row(row) and kind = "type-variable" and expectedArity = 2
     |
       actualArity = count(row.indexOf(";")) + 1 and
       actualArity != expectedArity and
@@ -499,7 +673,7 @@ module ModelOutput {
     or
     // Check names and arguments of access path tokens
     exists(AccessPath path, AccessPathToken token |
-      isRelevantFullPath(_, _, path) and
+      (isRelevantFullPath(_, _, path) or typeVariableModel(_, path)) and
       token = path.getToken(_)
     |
       not isValidTokenNameInIdentifyingAccessPath(token.getName()) and

python/ql/lib/semmle/python/frameworks/data/internal/ApiGraphModelsSpecific.qll

@@ -20,7 +20,6 @@
  */
 
 private import python as PY
-private import semmle.python.dataflow.new.DataFlow
 private import ApiGraphModels
 import semmle.python.ApiGraphs::API as API
 
@@ -28,6 +27,7 @@ class Unit = PY::Unit;
 
 // Re-export libraries needed by ApiGraphModels.qll
 import semmle.python.dataflow.new.internal.AccessPathSyntax as AccessPathSyntax
+import semmle.python.dataflow.new.DataFlow::DataFlow as DataFlow
 private import AccessPathSyntax
 
 /**
@@ -37,11 +37,12 @@ predicate isPackageUsed(string package) { API::moduleImportExists(package) }
 
 /** Gets a Python-specific interpretation of the `(package, type, path)` tuple after resolving the first `n` access path tokens. */
 bindingset[package, type, path]
-API::Node getExtraNodeFromPath(string package, string type, AccessPath path, int n) {
+API::Node getExtraNodeFromPath(string package, string type, AccessPath path, int n) { none() }
+
+/** Gets a Python-specific interpretation of the `(package, type)` tuple. */
+API::Node getExtraNodeFromType(string package, string type) {
   type = "" and
-  n = 0 and
-  result = API::moduleImport(package) and
-  exists(path)
+  result = API::moduleImport(package)
 }
 
 /**

python/ql/lib/semmle/python/internal/ConceptsShared.qll

@@ -87,3 +87,70 @@ module Cryptography {
     predicate isWeak() { this = "ECB" }
   }
 }
+
+/** Provides classes for modeling HTTP-related APIs. */
+module Http {
+  /** Provides classes for modeling HTTP clients. */
+  module Client {
+    /**
+     * A data-flow node that makes an outgoing HTTP request.
+     *
+     * Extend this class to refine existing API models. If you want to model new APIs,
+     * extend `Http::Client::Request::Range` instead.
+     */
+    class Request extends DataFlow::Node instanceof Request::Range {
+      /**
+       * Gets a data-flow node that contributes to the URL of the request.
+       * Depending on the framework, a request may have multiple nodes which contribute to the URL.
+       */
+      DataFlow::Node getAUrlPart() { result = super.getAUrlPart() }
+
+      /** Gets a string that identifies the framework used for this request. */
+      string getFramework() { result = super.getFramework() }
+
+      /**
+       * Holds if this request is made using a mode that disables SSL/TLS
+       * certificate validation, where `disablingNode` represents the point at
+       * which the validation was disabled, and `argumentOrigin` represents the origin
+       * of the argument that disabled the validation (which could be the same node as
+       * `disablingNode`).
+       */
+      predicate disablesCertificateValidation(
+        DataFlow::Node disablingNode, DataFlow::Node argumentOrigin
+      ) {
+        super.disablesCertificateValidation(disablingNode, argumentOrigin)
+      }
+    }
+
+    /** Provides a class for modeling new HTTP requests. */
+    module Request {
+      /**
+       * A data-flow node that makes an outgoing HTTP request.
+       *
+       * Extend this class to model new APIs. If you want to refine existing API models,
+       * extend `Http::Client::Request` instead.
+       */
+      abstract class Range extends DataFlow::Node {
+        /**
+         * Gets a data-flow node that contributes to the URL of the request.
+         * Depending on the framework, a request may have multiple nodes which contribute to the URL.
+         */
+        abstract DataFlow::Node getAUrlPart();
+
+        /** Gets a string that identifies the framework used for this request. */
+        abstract string getFramework();
+
+        /**
+         * Holds if this request is made using a mode that disables SSL/TLS
+         * certificate validation, where `disablingNode` represents the point at
+         * which the validation was disabled, and `argumentOrigin` represents the origin
+         * of the argument that disabled the validation (which could be the same node as
+         * `disablingNode`).
+         */
+        abstract predicate disablesCertificateValidation(
+          DataFlow::Node disablingNode, DataFlow::Node argumentOrigin
+        );
+      }
+    }
+  }
+}

python/ql/lib/semmle/python/objects/TObject.qll

@@ -334,7 +334,7 @@ predicate call3(
 }
 
 bindingset[self, function]
-predicate method_binding(
+deprecated predicate method_binding(
   AttrNode instantiation, ObjectInternal self, CallableObjectInternal function,
   PointsToContext context
 ) {
@@ -357,7 +357,9 @@ predicate method_binding(
 
 /** Helper for method_binding */
 pragma[noinline]
-predicate receiver(AttrNode instantiation, PointsToContext context, ObjectInternal obj, string name) {
+deprecated predicate receiver(
+  AttrNode instantiation, PointsToContext context, ObjectInternal obj, string name
+) {
   PointsToInternal::pointsTo(instantiation.getObject(name), context, obj, _)
 }
 

python/ql/lib/semmle/python/pointsto/Base.qll

@@ -13,7 +13,7 @@ import semmle.python.essa.SsaDefinitions
 private import semmle.python.types.Builtins
 private import semmle.python.internal.CachedStages
 
-module BasePointsTo {
+deprecated module BasePointsTo {
   /** INTERNAL -- Use n.refersTo(value, _, origin) instead */
   pragma[noinline]
   predicate points_to(ControlFlowNode f, Object value, ControlFlowNode origin) {
@@ -27,13 +27,13 @@ module BasePointsTo {
 }
 
 /** Gets the kwargs parameter (`**kwargs`). In a function definition this is always a dict. */
-predicate kwargs_points_to(ControlFlowNode f, ClassObject cls) {
+deprecated predicate kwargs_points_to(ControlFlowNode f, ClassObject cls) {
   exists(Function func | func.getKwarg() = f.getNode()) and
   cls = theDictType()
 }
 
 /** Gets the varargs parameter (`*varargs`). In a function definition this is always a tuple. */
-predicate varargs_points_to(ControlFlowNode f, ClassObject cls) {
+deprecated predicate varargs_points_to(ControlFlowNode f, ClassObject cls) {
   exists(Function func | func.getVararg() = f.getNode()) and
   cls = theTupleType()
 }
@@ -45,7 +45,7 @@ predicate varargs_points_to(ControlFlowNode f, ClassObject cls) {
  *  This exists primarily for internal use. Use getAnInferredType() instead.
  */
 pragma[noinline]
-ClassObject simple_types(Object obj) {
+deprecated ClassObject simple_types(Object obj) {
   result = comprehension(obj.getOrigin())
   or
   result = collection_literal(obj.getOrigin())
@@ -59,7 +59,7 @@ ClassObject simple_types(Object obj) {
   obj = unknownValue() and result = theUnknownType()
 }
 
-private ClassObject comprehension(Expr e) {
+deprecated private ClassObject comprehension(Expr e) {
   e instanceof ListComp and result = theListType()
   or
   e instanceof SetComp and result = theSetType()
@@ -69,7 +69,7 @@ private ClassObject comprehension(Expr e) {
   e instanceof GeneratorExp and result = theGeneratorType()
 }
 
-private ClassObject collection_literal(Expr e) {
+deprecated private ClassObject collection_literal(Expr e) {
   e instanceof List and result = theListType()
   or
   e instanceof Set and result = theSetType()
@@ -79,7 +79,7 @@ private ClassObject collection_literal(Expr e) {
   e instanceof Tuple and result = theTupleType()
 }
 
-private int tuple_index_value(Object t, int i) {
+deprecated private int tuple_index_value(Object t, int i) {
   result = t.(TupleNode).getElement(i).getNode().(Num).getN().toInt()
   or
   exists(Object item |
@@ -89,7 +89,7 @@ private int tuple_index_value(Object t, int i) {
 }
 
 pragma[noinline]
-int version_tuple_value(Object t) {
+deprecated int version_tuple_value(Object t) {
   not exists(tuple_index_value(t, 1)) and result = tuple_index_value(t, 0) * 10
   or
   not exists(tuple_index_value(t, 2)) and
@@ -102,7 +102,7 @@ int version_tuple_value(Object t) {
 }
 
 /** Choose a version numbers that represent the extreme of supported versions. */
-private int major_minor() {
+deprecated private int major_minor() {
   if major_version() = 3
   then (
     result = 33 or result = 37
@@ -113,7 +113,7 @@ private int major_minor() {
 }
 
 /** Compares the given tuple object to both the maximum and minimum possible sys.version_info values */
-int version_tuple_compare(Object t) {
+deprecated int version_tuple_compare(Object t) {
   version_tuple_value(t) < major_minor() and result = -1
   or
   version_tuple_value(t) = major_minor() and result = 0
@@ -122,7 +122,7 @@ int version_tuple_compare(Object t) {
 }
 
 /** Holds if `cls` is a new-style class if it were to have no explicit base classes */
-predicate baseless_is_new_style(ClassObject cls) {
+deprecated predicate baseless_is_new_style(ClassObject cls) {
   cls.isBuiltin()
   or
   major_version() = 3 and exists(cls)
@@ -160,7 +160,7 @@ private predicate class_defines_name(Class cls, string name) {
 }
 
 /** Gets a return value CFG node, provided that is safe to track across returns */
-ControlFlowNode safe_return_node(PyFunctionObject func) {
+deprecated ControlFlowNode safe_return_node(PyFunctionObject func) {
   result = func.getAReturnedNode() and
   // Not a parameter
   not exists(Parameter p, SsaVariable pvar |
@@ -172,7 +172,7 @@ ControlFlowNode safe_return_node(PyFunctionObject func) {
 }
 
 /** Holds if it can be determined from the control flow graph alone that this function can never return */
-predicate function_can_never_return(FunctionObject func) {
+deprecated predicate function_can_never_return(FunctionObject func) {
   /*
    * A Python function never returns if it has no normal exits that are not dominated by a
    * call to a function which itself never returns.
@@ -188,7 +188,9 @@ predicate function_can_never_return(FunctionObject func) {
 
 /** Hold if outer contains inner, both are contained within a test and inner is a use is a plain use or an attribute lookup */
 pragma[noinline]
-predicate contains_interesting_expression_within_test(ControlFlowNode outer, ControlFlowNode inner) {
+deprecated predicate contains_interesting_expression_within_test(
+  ControlFlowNode outer, ControlFlowNode inner
+) {
   inner.isLoad() and
   exists(ControlFlowNode test |
     outer.getAChild*() = inner and
@@ -208,7 +210,7 @@ predicate test_contains(ControlFlowNode expr, ControlFlowNode use) {
 }
 
 /** Holds if `test` is a test (a branch), `use` is within that test and `def` is an edge from that test with `sense` */
-predicate refinement_test(
+deprecated predicate refinement_test(
   ControlFlowNode test, ControlFlowNode use, boolean sense, PyEdgeRefinement def
 ) {
   /*
@@ -224,7 +226,7 @@ predicate refinement_test(
 
 /** Holds if `f` is an import of the form `from .[...] import name` and the enclosing scope is an __init__ module */
 pragma[noinline]
-predicate live_import_from_dot_in_init(ImportMemberNode f, EssaVariable var) {
+deprecated predicate live_import_from_dot_in_init(ImportMemberNode f, EssaVariable var) {
   exists(string name |
     import_from_dot_in_init(f.getModule(name)) and
     var.getSourceVariable().getName() = name and
@@ -249,13 +251,13 @@ Object undefinedVariable() { py_special_objects(result, "_semmle_undefined_value
 /** Gets the pseudo-object representing an unknown value */
 Object unknownValue() { result.asBuiltin() = Builtin::unknown() }
 
-BuiltinCallable theTypeNewMethod() {
+deprecated BuiltinCallable theTypeNewMethod() {
   result.asBuiltin() = theTypeType().asBuiltin().getMember("__new__")
 }
 
 /** Gets the `value, cls, origin` that `f` would refer to if it has not been assigned some other value */
 pragma[noinline]
-predicate potential_builtin_points_to(
+deprecated predicate potential_builtin_points_to(
   NameNode f, Object value, ClassObject cls, ControlFlowNode origin
 ) {
   f.isGlobal() and
@@ -269,7 +271,7 @@ predicate potential_builtin_points_to(
 }
 
 pragma[noinline]
-predicate builtin_name_points_to(string name, Object value, ClassObject cls) {
+deprecated predicate builtin_name_points_to(string name, Object value, ClassObject cls) {
   value = Object::builtin(name) and cls.asBuiltin() = value.asBuiltin().getClass()
 }
 
@@ -331,7 +333,9 @@ module BaseFlow {
 }
 
 /** Points-to for syntactic elements where context is not relevant */
-predicate simple_points_to(ControlFlowNode f, Object value, ClassObject cls, ControlFlowNode origin) {
+deprecated predicate simple_points_to(
+  ControlFlowNode f, Object value, ClassObject cls, ControlFlowNode origin
+) {
   kwargs_points_to(f, cls) and value = f and origin = f
   or
   varargs_points_to(f, cls) and value = f and origin = f
@@ -347,7 +351,9 @@ predicate simple_points_to(ControlFlowNode f, Object value, ClassObject cls, Con
  * Holds if `bit` is a binary expression node with a bitwise operator.
  * Helper for `this_binary_expr_points_to`.
  */
-predicate bitwise_expression_node(BinaryExprNode bit, ControlFlowNode left, ControlFlowNode right) {
+deprecated predicate bitwise_expression_node(
+  BinaryExprNode bit, ControlFlowNode left, ControlFlowNode right
+) {
   exists(Operator op | op = bit.getNode().getOp() |
     op instanceof BitAnd or
     op instanceof BitOr or
@@ -357,13 +363,13 @@ predicate bitwise_expression_node(BinaryExprNode bit, ControlFlowNode left, Cont
   right = bit.getRight()
 }
 
-private Module theCollectionsAbcModule() {
+deprecated private Module theCollectionsAbcModule() {
   result.getName() = "_abcoll"
   or
   result.getName() = "_collections_abc"
 }
 
-ClassObject collectionsAbcClass(string name) {
+deprecated ClassObject collectionsAbcClass(string name) {
   exists(Class cls |
     result.getPyClass() = cls and
     cls.getName() = name and

python/ql/lib/semmle/python/pointsto/Filters.qll

@@ -13,7 +13,7 @@ predicate hasattr(CallNode c, ControlFlowNode obj, string attr) {
 }
 
 /** Holds if `c` is a call to `callable(obj)`. */
-predicate is_callable(CallNode c, ControlFlowNode obj) {
+deprecated predicate is_callable(CallNode c, ControlFlowNode obj) {
   c.getFunction().(NameNode).getId() = "callable" and
   obj = c.getArg(0)
 }
@@ -26,7 +26,7 @@ predicate isinstance(CallNode fc, ControlFlowNode cls, ControlFlowNode use) {
 }
 
 /** Holds if `c` is a call to `issubclass(use, cls)`. */
-predicate issubclass(CallNode fc, ControlFlowNode cls, ControlFlowNode use) {
+deprecated predicate issubclass(CallNode fc, ControlFlowNode cls, ControlFlowNode use) {
   fc.getFunction().(NameNode).getId() = "issubclass" and
   fc.getArg(0) = use and
   cls = fc.getArg(1)

python/ql/lib/semmle/python/pointsto/PointsToContext.qll

@@ -122,7 +122,7 @@ private newtype TPointsToContext =
   } or
   TObjectContext(SelfInstanceInternal object)
 
-module Context {
+deprecated module Context {
   PointsToContext forObject(ObjectInternal object) { result = TObjectContext(object) }
 }
 

python/ql/lib/semmle/python/protocols.qll

@@ -1,19 +1,19 @@
 import python
 
 /** Retained for backwards compatibility use ClassObject.isIterator() instead. */
-predicate is_iterator(ClassObject c) { c.isIterator() }
+deprecated predicate is_iterator(ClassObject c) { c.isIterator() }
 
 /** Retained for backwards compatibility use ClassObject.isIterable() instead. */
-predicate is_iterable(ClassObject c) { c.isIterable() }
+deprecated predicate is_iterable(ClassObject c) { c.isIterable() }
 
 /** Retained for backwards compatibility use ClassObject.isCollection() instead. */
-predicate is_collection(ClassObject c) { c.isCollection() }
+deprecated predicate is_collection(ClassObject c) { c.isCollection() }
 
 /** Retained for backwards compatibility use ClassObject.isMapping() instead. */
-predicate is_mapping(ClassObject c) { c.isMapping() }
+deprecated predicate is_mapping(ClassObject c) { c.isMapping() }
 
 /** Retained for backwards compatibility use ClassObject.isSequence() instead. */
-predicate is_sequence(ClassObject c) { c.isSequence() }
+deprecated predicate is_sequence(ClassObject c) { c.isSequence() }
 
 /** Retained for backwards compatibility use ClassObject.isContextManager() instead. */
-predicate is_context_manager(ClassObject c) { c.isContextManager() }
+deprecated predicate is_context_manager(ClassObject c) { c.isContextManager() }

python/ql/lib/semmle/python/security/OverlyLargeRangeQuery.qll

@@ -173,7 +173,7 @@ module RangePrinter {
   }
 
   /** Gets the number of parts we should print for a given `range`. */
-  private int parts(OverlyWideRange range) { result = 1 + strictcount(cutoff(range, _)) }
+  private int parts(OverlyWideRange range) { result = 1 + count(cutoff(range, _)) }
 
   /** Holds if the given part of a range should span from `low` to `high`. */
   private predicate part(OverlyWideRange range, int part, string low, string high) {
@@ -238,8 +238,13 @@ module RangePrinter {
 
 /** Gets a char range that is overly large because of `reason`. */
 RegExpCharacterRange getABadRange(string reason, int priority) {
+  result instanceof OverlyWideRange and
   priority = 0 and
-  reason = "is equivalent to " + result.(OverlyWideRange).printEquivalent()
+  exists(string equiv | equiv = result.(OverlyWideRange).printEquivalent() |
+    if equiv.length() <= 50
+    then reason = "is equivalent to " + equiv
+    else reason = "is equivalent to " + equiv.substring(0, 50) + "..."
+  )
   or
   priority = 1 and
   exists(RegExpCharacterRange other |

python/ql/lib/semmle/python/security/regexp/ExponentialBackTracking.qll

@@ -115,6 +115,7 @@ private newtype TStatePair =
 private int rankState(State state) {
   state =
     rank[result](State s, Location l |
+      stateInsideBacktracking(s) and
       l = s.getRepr().getLocation()
     |
       s order by l.getStartLine(), l.getStartColumn(), s.toString()

python/ql/lib/semmle/python/security/regexp/NfaUtils.qll

@@ -93,8 +93,6 @@ class RegExpRoot extends RegExpTerm {
    * Holds if this root term is relevant to the ReDoS analysis.
    */
   predicate isRelevant() {
-    // there is at least one repetition
-    getRoot(any(InfiniteRepetitionQuantifier q)) = this and
     // is actually used as a RegExp
     this.isUsedAsRegExp() and
     // not excluded for library specific reasons
@@ -877,6 +875,107 @@ predicate isStartState(State state) {
  */
 signature predicate isCandidateSig(State state, string pump);
 
+/**
+ * Holds if `state` is a candidate for ReDoS.
+ */
+signature predicate isCandidateSig(State state);
+
+/**
+ * Predicates for constructing a prefix string that leads to a given state.
+ */
+module PrefixConstruction<isCandidateSig/1 isCandidate> {
+  /**
+   * Holds if `state` is the textually last start state for the regular expression.
+   */
+  private predicate lastStartState(RelevantState state) {
+    exists(RegExpRoot root |
+      state =
+        max(RelevantState s, Location l |
+          isStartState(s) and
+          getRoot(s.getRepr()) = root and
+          l = s.getRepr().getLocation()
+        |
+          s
+          order by
+            l.getStartLine(), l.getStartColumn(), s.getRepr().toString(), l.getEndColumn(),
+            l.getEndLine()
+        )
+    )
+  }
+
+  /**
+   * Holds if there exists any transition (Epsilon() or other) from `a` to `b`.
+   */
+  private predicate existsTransition(State a, State b) { delta(a, _, b) }
+
+  /**
+   * Gets the minimum number of transitions it takes to reach `state` from the `start` state.
+   */
+  int prefixLength(State start, State state) =
+    shortestDistances(lastStartState/1, existsTransition/2)(start, state, result)
+
+  /**
+   * Gets the minimum number of transitions it takes to reach `state` from the start state.
+   */
+  private int lengthFromStart(State state) { result = prefixLength(_, state) }
+
+  /**
+   * Gets a string for which the regular expression will reach `state`.
+   *
+   * Has at most one result for any given `state`.
+   * This predicate will not always have a result even if there is a ReDoS issue in
+   * the regular expression.
+   */
+  string prefix(State state) {
+    lastStartState(state) and
+    result = ""
+    or
+    // the search stops past the last redos candidate state.
+    lengthFromStart(state) <= max(lengthFromStart(any(State s | isCandidate(s)))) and
+    exists(State prev |
+      // select a unique predecessor (by an arbitrary measure)
+      prev =
+        min(State s, Location loc |
+          lengthFromStart(s) = lengthFromStart(state) - 1 and
+          loc = s.getRepr().getLocation() and
+          delta(s, _, state)
+        |
+          s
+          order by
+            loc.getStartLine(), loc.getStartColumn(), loc.getEndLine(), loc.getEndColumn(),
+            s.getRepr().toString()
+        )
+    |
+      // greedy search for the shortest prefix
+      result = prefix(prev) and delta(prev, Epsilon(), state)
+      or
+      not delta(prev, Epsilon(), state) and
+      result = prefix(prev) + getCanonicalEdgeChar(prev, state)
+    )
+  }
+
+  /**
+   * Gets a canonical char for which there exists a transition from `prev` to `next` in the NFA.
+   */
+  private string getCanonicalEdgeChar(State prev, State next) {
+    result =
+      min(string c | delta(prev, any(InputSymbol symbol | c = intersect(Any(), symbol)), next))
+  }
+
+  /** A state within a regular expression that contains a candidate state. */
+  class RelevantState instanceof State {
+    RelevantState() {
+      exists(State s | isCandidate(s) | getRoot(s.getRepr()) = getRoot(this.getRepr()))
+    }
+
+    /** Gets a string representation for this state in a regular expression. */
+    string toString() { result = State.super.toString() }
+
+    /** Gets the term represented by this state. */
+    RegExpTerm getRepr() { result = State.super.getRepr() }
+  }
+}
+
 /**
  * A module for pruning candidate ReDoS states.
  * The candidates are specified by the `isCandidate` signature predicate.
@@ -910,95 +1009,11 @@ module ReDoSPruning<isCandidateSig/2 isCandidate> {
   /** Gets a state that can reach the `accept-any` state using only epsilon steps. */
   private State acceptsAnySuffix() { epsilonSucc*(result) = AcceptAnySuffix(_) }
 
-  /**
-   * Predicates for constructing a prefix string that leads to a given state.
-   */
-  private module PrefixConstruction {
-    /**
-     * Holds if `state` is the textually last start state for the regular expression.
-     */
-    private predicate lastStartState(State state) {
-      exists(RegExpRoot root |
-        state =
-          max(State s, Location l |
-            s = stateInPumpableRegexp() and
-            isStartState(s) and
-            getRoot(s.getRepr()) = root and
-            l = s.getRepr().getLocation()
-          |
-            s
-            order by
-              l.getStartLine(), l.getStartColumn(), s.getRepr().toString(), l.getEndColumn(),
-              l.getEndLine()
-          )
-      )
-    }
+  predicate isCandidateState(State s) { isReDoSCandidate(s, _) }
 
-    /**
-     * Holds if there exists any transition (Epsilon() or other) from `a` to `b`.
-     */
-    private predicate existsTransition(State a, State b) { delta(a, _, b) }
+  import PrefixConstruction<isCandidateState/1> as Prefix
 
-    /**
-     * Gets the minimum number of transitions it takes to reach `state` from the `start` state.
-     */
-    int prefixLength(State start, State state) =
-      shortestDistances(lastStartState/1, existsTransition/2)(start, state, result)
-
-    /**
-     * Gets the minimum number of transitions it takes to reach `state` from the start state.
-     */
-    private int lengthFromStart(State state) { result = prefixLength(_, state) }
-
-    /**
-     * Gets a string for which the regular expression will reach `state`.
-     *
-     * Has at most one result for any given `state`.
-     * This predicate will not always have a result even if there is a ReDoS issue in
-     * the regular expression.
-     */
-    string prefix(State state) {
-      lastStartState(state) and
-      result = ""
-      or
-      // the search stops past the last redos candidate state.
-      lengthFromStart(state) <= max(lengthFromStart(any(State s | isReDoSCandidate(s, _)))) and
-      exists(State prev |
-        // select a unique predecessor (by an arbitrary measure)
-        prev =
-          min(State s, Location loc |
-            lengthFromStart(s) = lengthFromStart(state) - 1 and
-            loc = s.getRepr().getLocation() and
-            delta(s, _, state)
-          |
-            s
-            order by
-              loc.getStartLine(), loc.getStartColumn(), loc.getEndLine(), loc.getEndColumn(),
-              s.getRepr().toString()
-          )
-      |
-        // greedy search for the shortest prefix
-        result = prefix(prev) and delta(prev, Epsilon(), state)
-        or
-        not delta(prev, Epsilon(), state) and
-        result = prefix(prev) + getCanonicalEdgeChar(prev, state)
-      )
-    }
-
-    /**
-     * Gets a canonical char for which there exists a transition from `prev` to `next` in the NFA.
-     */
-    private string getCanonicalEdgeChar(State prev, State next) {
-      result =
-        min(string c | delta(prev, any(InputSymbol symbol | c = intersect(Any(), symbol)), next))
-    }
-
-    /** Gets a state within a regular expression that has a pumpable state. */
-    pragma[noinline]
-    State stateInPumpableRegexp() {
-      exists(State s | isReDoSCandidate(s, _) | getRoot(s.getRepr()) = getRoot(result.getRepr()))
-    }
-  }
+  class RelevantState = Prefix::RelevantState;
 
   /**
    * Predicates for testing the presence of a rejecting suffix.
@@ -1018,8 +1033,6 @@ module ReDoSPruning<isCandidateSig/2 isCandidate> {
    * using epsilon transitions. But any attempt at repeating `w` will end in a state that accepts all suffixes.
    */
   private module SuffixConstruction {
-    import PrefixConstruction
-
     /**
      * Holds if all states reachable from `fork` by repeating `w`
      * are likely rejectable by appending some suffix.
@@ -1035,32 +1048,26 @@ module ReDoSPruning<isCandidateSig/2 isCandidate> {
      * This predicate might find impossible suffixes when searching for suffixes of length > 1, which can cause FPs.
      */
     pragma[noinline]
-    private predicate isLikelyRejectable(State s) {
-      s = stateInPumpableRegexp() and
-      (
-        // exists a reject edge with some char.
-        hasRejectEdge(s)
-        or
-        hasEdgeToLikelyRejectable(s)
-        or
-        // stopping here is rejection
-        isRejectState(s)
-      )
+    private predicate isLikelyRejectable(RelevantState s) {
+      // exists a reject edge with some char.
+      hasRejectEdge(s)
+      or
+      hasEdgeToLikelyRejectable(s)
+      or
+      // stopping here is rejection
+      isRejectState(s)
     }
 
     /**
      * Holds if `s` is not an accept state, and there is no epsilon transition to an accept state.
      */
-    predicate isRejectState(State s) {
-      s = stateInPumpableRegexp() and not epsilonSucc*(s) = Accept(_)
-    }
+    predicate isRejectState(RelevantState s) { not epsilonSucc*(s) = Accept(_) }
 
     /**
      * Holds if there is likely a non-empty suffix leading to rejection starting in `s`.
      */
     pragma[noopt]
-    predicate hasEdgeToLikelyRejectable(State s) {
-      s = stateInPumpableRegexp() and
+    predicate hasEdgeToLikelyRejectable(RelevantState s) {
       // all edges (at least one) with some char leads to another state that is rejectable.
       // the `next` states might not share a common suffix, which can cause FPs.
       exists(string char | char = hasEdgeToLikelyRejectableHelper(s) |
@@ -1075,8 +1082,7 @@ module ReDoSPruning<isCandidateSig/2 isCandidate> {
      * and `s` has not been found to be rejectable by `hasRejectEdge` or `isRejectState`.
      */
     pragma[noinline]
-    private string hasEdgeToLikelyRejectableHelper(State s) {
-      s = stateInPumpableRegexp() and
+    private string hasEdgeToLikelyRejectableHelper(RelevantState s) {
       not hasRejectEdge(s) and
       not isRejectState(s) and
       deltaClosedChar(s, result, _)
@@ -1087,9 +1093,7 @@ module ReDoSPruning<isCandidateSig/2 isCandidate> {
      * along epsilon edges, such that there is a transition from
      * `prev` to `next` that the character symbol `char`.
      */
-    predicate deltaClosedChar(State prev, string char, State next) {
-      prev = stateInPumpableRegexp() and
-      next = stateInPumpableRegexp() and
+    predicate deltaClosedChar(RelevantState prev, string char, RelevantState next) {
       deltaClosed(prev, getAnInputSymbolMatchingRelevant(char), next)
     }
 
@@ -1099,18 +1103,28 @@ module ReDoSPruning<isCandidateSig/2 isCandidate> {
       result = getAnInputSymbolMatching(char)
     }
 
+    pragma[noinline]
+    RegExpRoot relevantRoot() {
+      exists(RegExpTerm term, State s |
+        s.getRepr() = term and isCandidateState(s) and result = term.getRootTerm()
+      )
+    }
+
     /**
      * Gets a char used for finding possible suffixes inside `root`.
      */
     pragma[noinline]
     private string relevant(RegExpRoot root) {
-      exists(ascii(result)) and exists(root)
-      or
-      exists(InputSymbol s | belongsTo(s, root) | result = intersect(s, _))
-      or
-      // The characters from `hasSimpleRejectEdge`. Only `\n` is really needed (as `\n` is not in the `ascii` relation).
-      // The three chars must be kept in sync with `hasSimpleRejectEdge`.
-      result = ["|", "\n", "Z"] and exists(root)
+      root = relevantRoot() and
+      (
+        exists(ascii(result)) and exists(root)
+        or
+        exists(InputSymbol s | belongsTo(s, root) | result = intersect(s, _))
+        or
+        // The characters from `hasSimpleRejectEdge`. Only `\n` is really needed (as `\n` is not in the `ascii` relation).
+        // The three chars must be kept in sync with `hasSimpleRejectEdge`.
+        result = ["|", "\n", "Z"] and exists(root)
+      )
     }
 
     /**
@@ -1208,12 +1222,12 @@ module ReDoSPruning<isCandidateSig/2 isCandidate> {
   predicate hasReDoSResult(RegExpTerm t, string pump, State s, string prefixMsg) {
     isReDoSAttackable(t, pump, s) and
     (
-      prefixMsg = "starting with '" + escape(PrefixConstruction::prefix(s)) + "' and " and
-      not PrefixConstruction::prefix(s) = ""
+      prefixMsg = "starting with '" + escape(Prefix::prefix(s)) + "' and " and
+      not Prefix::prefix(s) = ""
       or
-      PrefixConstruction::prefix(s) = "" and prefixMsg = ""
+      Prefix::prefix(s) = "" and prefixMsg = ""
       or
-      not exists(PrefixConstruction::prefix(s)) and prefixMsg = ""
+      not exists(Prefix::prefix(s)) and prefixMsg = ""
     )
   }
 

python/ql/src/CHANGELOG.md

@@ -1,3 +1,10 @@
+## 0.4.2
+
+### New Queries
+
+* Added a new query, `py/suspicious-regexp-range`, to detect character ranges in regular expressions that seem to match 
+  too many characters.
+
 ## 0.4.1
 
 ## 0.4.0

python/ql/src/Exceptions/EmptyExcept.ql

@@ -12,6 +12,7 @@
  */
 
 import python
+import semmle.python.ApiGraphs
 
 predicate empty_except(ExceptStmt ex) {
   not exists(Stmt s | s = ex.getAStmt() and not s instanceof Pass)
@@ -28,7 +29,7 @@ predicate no_comment(ExceptStmt ex) {
 }
 
 predicate non_local_control_flow(ExceptStmt ex) {
-  ex.getType().pointsTo(ClassValue::stopIteration())
+  ex.getType() = API::builtin("StopIteration").getAValueReachableFromSource().asExpr()
 }
 
 predicate try_has_normal_exit(Try try) {
@@ -61,27 +62,32 @@ predicate subscript(Stmt s) {
   s.(Delete).getATarget() instanceof Subscript
 }
 
-predicate encode_decode(Call ex, ClassValue type) {
+predicate encode_decode(Call ex, Expr type) {
   exists(string name | ex.getFunc().(Attribute).getName() = name |
-    name = "encode" and type = ClassValue::unicodeEncodeError()
+    name = "encode" and
+    type = API::builtin("UnicodeEncodeError").getAValueReachableFromSource().asExpr()
     or
-    name = "decode" and type = ClassValue::unicodeDecodeError()
+    name = "decode" and
+    type = API::builtin("UnicodeDecodeError").getAValueReachableFromSource().asExpr()
   )
 }
 
-predicate small_handler(ExceptStmt ex, Stmt s, ClassValue type) {
+predicate small_handler(ExceptStmt ex, Stmt s, Expr type) {
   not exists(ex.getTry().getStmt(1)) and
   s = ex.getTry().getStmt(0) and
-  ex.getType().pointsTo(type)
+  ex.getType() = type
 }
 
 predicate focussed_handler(ExceptStmt ex) {
-  exists(Stmt s, ClassValue type | small_handler(ex, s, type) |
-    subscript(s) and type.getASuperType() = ClassValue::lookupError()
+  exists(Stmt s, Expr type | small_handler(ex, s, type) |
+    subscript(s) and
+    type = API::builtin("IndexError").getASubclass*().getAValueReachableFromSource().asExpr()
     or
-    attribute_access(s) and type = ClassValue::attributeError()
+    attribute_access(s) and
+    type = API::builtin("AttributeError").getAValueReachableFromSource().asExpr()
     or
-    s.(ExprStmt).getValue() instanceof Name and type = ClassValue::nameError()
+    s.(ExprStmt).getValue() instanceof Name and
+    type = API::builtin("NameError").getAValueReachableFromSource().asExpr()
     or
     encode_decode(s.(ExprStmt).getValue(), type)
   )

python/ql/src/Functions/ModificationOfParameterWithDefault.ql

@@ -19,5 +19,5 @@ from
   ModificationOfParameterWithDefault::Configuration config, DataFlow::PathNode source,
   DataFlow::PathNode sink
 where config.hasFlowPath(source, sink)
-select sink.getNode(), source, sink, "$@ flows to here and is mutated.", source.getNode(),
-  "Default value"
+select sink.getNode(), source, sink, "This expression mutates $@.", source.getNode(),
+  "a default value"

python/ql/src/Lexical/CommentedOutCode.ql

@@ -16,4 +16,4 @@ import Lexical.CommentedOutCode
 
 from CommentedOutCodeBlock c
 where not c.maybeExampleCode()
-select c, "These comments appear to contain commented-out code."
+select c, "This comment appears to contain commented-out code."

python/ql/src/Security/CWE-022/TarSlip.ql

@@ -18,5 +18,5 @@ import DataFlow::PathGraph
 
 from Configuration config, DataFlow::PathNode source, DataFlow::PathNode sink
 where config.hasFlowPath(source, sink)
-select sink.getNode(), source, sink, "Extraction of tarfile from $@", source.getNode(),
+select sink.getNode(), source, sink, "This file extraction depends on $@", source.getNode(),
   "a potentially untrusted source"

python/ql/src/Security/CWE-078/CommandInjection.ql

@@ -20,5 +20,5 @@ import DataFlow::PathGraph
 
 from Configuration config, DataFlow::PathNode source, DataFlow::PathNode sink
 where config.hasFlowPath(source, sink)
-select sink.getNode(), source, sink, "This command depends on $@.", source.getNode(),
+select sink.getNode(), source, sink, "This command line depends on $@.", source.getNode(),
   "a user-provided value"

python/ql/src/Security/CWE-090/LdapInjection.ql

@@ -23,6 +23,5 @@ where
   or
   any(FilterConfiguration filterConfig).hasFlowPath(source, sink) and
   parameterName = "filter"
-select sink.getNode(), source, sink,
-  "$@ LDAP query parameter (" + parameterName + ") comes from $@.", sink.getNode(), "This",
-  source.getNode(), "a user-provided value"
+select sink.getNode(), source, sink, "$@ depends on $@.", sink.getNode(),
+  "LDAP query parameter (" + parameterName + ")", source.getNode(), "a user-provided value"

python/ql/src/Security/CWE-094/CodeInjection.ql

@@ -20,5 +20,5 @@ import DataFlow::PathGraph
 
 from Configuration config, DataFlow::PathNode source, DataFlow::PathNode sink
 where config.hasFlowPath(source, sink)
-select sink.getNode(), source, sink, "$@ flows to here and is interpreted as code.",
-  source.getNode(), "A user-provided value"
+select sink.getNode(), source, sink, "This code execution depends on $@.", source.getNode(),
+  "a user-provided value"

python/ql/src/Security/CWE-117/LogInjection.ql

@@ -17,5 +17,5 @@ import DataFlow::PathGraph
 
 from Configuration config, DataFlow::PathNode source, DataFlow::PathNode sink
 where config.hasFlowPath(source, sink)
-select sink.getNode(), source, sink, "$@ flows to log entry.", source.getNode(),
-  "User-provided value"
+select sink.getNode(), source, sink, "This log entry depends on $@.", source.getNode(),
+  "a user-provided value"

python/ql/src/Security/CWE-209/StackTraceExposure.ql

@@ -19,5 +19,6 @@ import DataFlow::PathGraph
 
 from Configuration config, DataFlow::PathNode source, DataFlow::PathNode sink
 where config.hasFlowPath(source, sink)
-select sink.getNode(), source, sink, "$@ may be exposed to an external user", source.getNode(),
-  "Error information"
+select sink.getNode(), source, sink,
+  "$@ flows to this location and may be exposed to an external user.", source.getNode(),
+  "Stack trace information"

python/ql/src/Security/CWE-295/RequestWithoutValidation.ql

@@ -13,7 +13,6 @@
 import python
 private import semmle.python.dataflow.new.DataFlow
 private import semmle.python.Concepts
-private import semmle.python.ApiGraphs
 
 from
   HTTP::Client::Request request, DataFlow::Node disablingNode, DataFlow::Node origin, string ending

python/ql/src/Security/CWE-502/UnsafeDeserialization.ql

@@ -18,4 +18,5 @@ import DataFlow::PathGraph
 
 from Configuration config, DataFlow::PathNode source, DataFlow::PathNode sink
 where config.hasFlowPath(source, sink)
-select sink.getNode(), source, sink, "Deserializing of $@.", source.getNode(), "untrusted input"
+select sink.getNode(), source, sink, "Unsafe deserialization depends on $@.", source.getNode(),
+  "a user-provided value"

python/ql/src/Security/CWE-601/UrlRedirect.ql

@@ -18,5 +18,5 @@ import DataFlow::PathGraph
 
 from Configuration config, DataFlow::PathNode source, DataFlow::PathNode sink
 where config.hasFlowPath(source, sink)
-select sink.getNode(), source, sink, "Untrusted URL redirection due to $@.", source.getNode(),
-  "A user-provided value"
+select sink.getNode(), source, sink, "Untrusted URL redirection depends on $@.", source.getNode(),
+  "a user-provided value"

python/ql/src/Security/CWE-611/Xxe.ql

@@ -19,5 +19,5 @@ import DataFlow::PathGraph
 from Configuration cfg, DataFlow::PathNode source, DataFlow::PathNode sink
 where cfg.hasFlowPath(source, sink)
 select sink.getNode(), source, sink,
-  "A $@ is parsed as XML without guarding against external entity expansion.", source.getNode(),
-  "user-provided value"
+  "XML parsing depends on $@ without guarding against external entity expansion.", source.getNode(),
+  "a user-provided value"

python/ql/src/Security/CWE-643/XpathInjection.ql

@@ -17,4 +17,5 @@ import DataFlow::PathGraph
 
 from Configuration config, DataFlow::PathNode source, DataFlow::PathNode sink
 where config.hasFlowPath(source, sink)
-select sink, source, sink, "This Xpath query depends on $@.", source, "a user-provided value"
+select sink.getNode(), source, sink, "XPath expression depends on $@.", source.getNode(),
+  "a user-provided value"

python/ql/src/Security/CWE-730/PolynomialReDoS.ql

@@ -4,6 +4,7 @@
  *              to match may be vulnerable to denial-of-service attacks.
  * @kind path-problem
  * @problem.severity warning
+ * @security-severity 7.5
  * @precision high
  * @id py/polynomial-redos
  * @tags security

python/ql/src/Security/CWE-730/ReDoS.ql

@@ -5,6 +5,7 @@
  *              attacks.
  * @kind problem
  * @problem.severity error
+ * @security-severity 7.5
  * @precision high
  * @id py/redos
  * @tags security

python/ql/src/Security/CWE-730/RegexInjection.ql

@@ -5,6 +5,7 @@
  *              exponential time on certain inputs.
  * @kind path-problem
  * @problem.severity error
+ * @security-severity 7.5
  * @precision high
  * @id py/regex-injection
  * @tags security
@@ -23,6 +24,6 @@ from
 where
   config.hasFlowPath(source, sink) and
   regexExecution = sink.getNode().(Sink).getRegexExecution()
-select sink.getNode(), source, sink,
-  "$@ regular expression is constructed from a $@ and executed by $@.", sink.getNode(), "This",
-  source.getNode(), "user-provided value", regexExecution, regexExecution.getName()
+select sink.getNode(), source, sink, "$@ depends on $@ and executed by $@.", sink.getNode(),
+  "This regular expression", source.getNode(), "a user-provided value", regexExecution,
+  regexExecution.getName()

python/ql/src/Security/CWE-776/XmlBomb.ql

@@ -19,5 +19,5 @@ import DataFlow::PathGraph
 from Configuration cfg, DataFlow::PathNode source, DataFlow::PathNode sink
 where cfg.hasFlowPath(source, sink)
 select sink.getNode(), source, sink,
-  "A $@ is parsed as XML without guarding against uncontrolled entity expansion.", source.getNode(),
-  "user-provided value"
+  "XML parsing depends on $@ without guarding against uncontrolled entity expansion.",
+  source.getNode(), "a user-provided value"

python/ql/src/Statements/UnreachableCode.ql

@@ -61,4 +61,4 @@ predicate reportable_unreachable(Stmt s) {
 
 from Stmt s
 where reportable_unreachable(s)
-select s, "Unreachable statement."
+select s, "This statement is unreachable."

python/ql/src/Variables/UnusedLocalVariable.ql

@@ -43,4 +43,4 @@ where
   unused_local(unused, v) and
   // If unused is part of a tuple, count it as unused if all elements of that tuple are unused.
   forall(Name el | el = unused.getParentNode().(Tuple).getAnElt() | unused_local(el, _))
-select unused, "The value assigned to local variable '" + v.getId() + "' is never used."
+select unused, "Variable " + v.getId() + " is not used"

python/ql/src/change-notes/2022-08-23-alert-messages.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* The alert message of many queries have been changed to make the message consistent with other languages.

python/ql/src/change-notes/2022-08-29-missing-security-severity.md

@@ -0,0 +1,4 @@
+---
+category: queryMetadata
+---
+* Added the `security-severity` tag the `py/redos`, `py/polynomial-redos`, and `py/regex-injection` queries.

python/ql/src/change-notes/released/0.4.2.md

@@ -1,5 +1,6 @@
----
-category: newQuery
----
+## 0.4.2
+
+### New Queries
+
 * Added a new query, `py/suspicious-regexp-range`, to detect character ranges in regular expressions that seem to match 
   too many characters.

python/ql/src/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.4.1
+lastReleaseVersion: 0.4.2

python/ql/src/experimental/semmle/python/frameworks/Django.qll

@@ -22,11 +22,14 @@ private module ExperimentalPrivateDjango {
 
       module Request {
         module HttpRequest {
-          class DjangoGETParameter extends DataFlow::Node, RemoteFlowSource::Range {
-            DjangoGETParameter() { this = request().getMember("GET").getMember("get").getACall() }
+          class DjangoGetParameter extends DataFlow::Node, RemoteFlowSource::Range {
+            DjangoGetParameter() { this = request().getMember("GET").getMember("get").getACall() }
 
             override string getSourceType() { result = "django.http.request.GET.get" }
           }
+
+          /** DEPRECATED: Alias for DjangoGetParameter */
+          deprecated class DjangoGETParameter = DjangoGetParameter;
         }
       }
 

python/ql/src/experimental/semmle/python/frameworks/LDAP.qll

@@ -126,9 +126,9 @@ private module Ldap {
           (
             // ldap_connection.start_tls_s()
             // see https://www.python-ldap.org/en/python-ldap-3.3.0/reference/ldap.html#ldap.LDAPObject.start_tls_s
-            exists(DataFlow::MethodCallNode startTLS |
-              startTLS.getObject().getALocalSource() = initialize and
-              startTLS.getMethodName() = "start_tls_s"
+            exists(DataFlow::MethodCallNode startTls |
+              startTls.getObject().getALocalSource() = initialize and
+              startTls.getMethodName() = "start_tls_s"
             )
             or
             // ldap_connection.set_option(ldap.OPT_X_TLS_%s, True)
@@ -234,9 +234,9 @@ private module Ldap {
         or
         // ldap_connection.start_tls_s()
         // see https://www.python-ldap.org/en/python-ldap-3.3.0/reference/ldap.html#ldap.LDAPObject.start_tls_s
-        exists(DataFlow::MethodCallNode startTLS |
-          startTLS.getMethodName() = "start_tls_s" and
-          startTLS.getObject().getALocalSource() = this
+        exists(DataFlow::MethodCallNode startTls |
+          startTls.getMethodName() = "start_tls_s" and
+          startTls.getObject().getALocalSource() = this
         )
       }
 

python/ql/src/experimental/semmle/python/libraries/SmtpLib.qll

@@ -31,8 +31,8 @@ module SmtpLib {
    * argument. Used because of the impossibility to get local source nodes from `_subparts`'
    * `(List|Tuple)` elements.
    */
-  private class SMTPMessageConfig extends TaintTracking2::Configuration {
-    SMTPMessageConfig() { this = "SMTPMessageConfig" }
+  private class SmtpMessageConfig extends TaintTracking2::Configuration {
+    SmtpMessageConfig() { this = "SMTPMessageConfig" }
 
     override predicate isSource(DataFlow::Node source) { source = mimeText(_) }
 
@@ -87,7 +87,7 @@ module SmtpLib {
         sink =
           [sendCall.getArg(2), sendCall.getArg(2).(DataFlow::MethodCallNode).getObject()]
               .getALocalSource() and
-        any(SMTPMessageConfig a)
+        any(SmtpMessageConfig a)
             .hasFlow(source, sink.(DataFlow::CallCfgNode).getArgByName("_subparts"))
         or
         // via .attach()
@@ -117,7 +117,7 @@ module SmtpLib {
    * * `sub` would be `message["Subject"]` (`Subscript`)
    * * `result` would be `"multipart test"`
    */
-  private DataFlow::Node getSMTPSubscriptByIndex(DataFlow::CallCfgNode sendCall, string index) {
+  private DataFlow::Node getSmtpSubscriptByIndex(DataFlow::CallCfgNode sendCall, string index) {
     exists(DefinitionNode def, Subscript sub |
       sub = def.getNode() and
       DataFlow::exprNode(sub.getObject()).getALocalSource() =
@@ -163,15 +163,15 @@ module SmtpLib {
     override DataFlow::Node getHtmlBody() { result = getSmtpMessage(this, "html") }
 
     override DataFlow::Node getTo() {
-      result in [this.getArg(1), getSMTPSubscriptByIndex(this, "To")]
+      result in [this.getArg(1), getSmtpSubscriptByIndex(this, "To")]
     }
 
     override DataFlow::Node getFrom() {
-      result in [this.getArg(0), getSMTPSubscriptByIndex(this, "From")]
+      result in [this.getArg(0), getSmtpSubscriptByIndex(this, "From")]
     }
 
     override DataFlow::Node getSubject() {
-      result in [this.getArg(2), getSMTPSubscriptByIndex(this, "Subject")]
+      result in [this.getArg(2), getSmtpSubscriptByIndex(this, "Subject")]
     }
   }
 }

python/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/python-queries
-version: 0.4.2-dev
+version: 0.4.3-dev
 groups: 
   - python
   - queries

python/ql/test/experimental/dataflow/TestUtil/NormalDataflowTest.qll

@@ -13,7 +13,7 @@ class DataFlowTest extends FlowTest {
   }
 }
 
-query predicate missingAnnotationOnSINK(Location location, string error, string element) {
+query predicate missingAnnotationOnSink(Location location, string error, string element) {
   error = "ERROR, you should add `# $ MISSING: flow` annotation" and
   exists(DataFlow::Node sink |
     exists(DataFlow::CallCfgNode call |
@@ -31,3 +31,6 @@ query predicate missingAnnotationOnSINK(Location location, string error, string
     )
   )
 }
+
+/** DEPRECATED: Alias for missingAnnotationOnSink */
+deprecated predicate missingAnnotationOnSINK = missingAnnotationOnSink/3;

python/ql/test/experimental/dataflow/coverage/NormalDataflowTest.expected

@@ -1,2 +1,2 @@
-missingAnnotationOnSINK
+missingAnnotationOnSink
 failures

python/ql/test/experimental/dataflow/fieldflow/NormalDataflowTest.expected

@@ -1,2 +1,2 @@
-missingAnnotationOnSINK
+missingAnnotationOnSink
 failures

python/ql/test/experimental/dataflow/match/NormalDataflowTest.expected

@@ -1,2 +1,2 @@
-missingAnnotationOnSINK
+missingAnnotationOnSink
 failures

python/ql/test/experimental/dataflow/strange-pointsto-interaction-investigation/README.md

@@ -39,7 +39,7 @@ diff '--color=auto' -u -r test-1-normal/NormalDataflowTest.expected test-5-max-i
 --- test-1-normal/NormalDataflowTest.expected	2022-02-27 10:33:00.603882599 +0100
 +++ test-5-max-import-depth-3/NormalDataflowTest.expected	2022-02-28 10:10:08.930743800 +0100
 @@ -1,2 +1,3 @@
- missingAnnotationOnSINK
+ missingAnnotationOnSink
  failures
 +| ../src/urandom_problem.py:43:6:43:8 | ControlFlowNode for foo | Fixed missing result:flow="SOURCE, l:-15 -> foo" |
 diff '--color=auto' -u -r test-1-normal/options test-5-max-import-depth-3/options
@@ -88,7 +88,7 @@ diff '--color=auto' -u -r test-4-max-import-depth-100/NormalDataflowTest.expecte
 --- test-4-max-import-depth-100/NormalDataflowTest.expected     2022-02-28 10:10:02.206608379 +0100
 +++ test-6-max-import-depth-2/NormalDataflowTest.expected       2022-02-28 10:10:13.882716665 +0100
 @@ -1,3 +1,5 @@
- missingAnnotationOnSINK
+ missingAnnotationOnSink
 +| ../src/isfile_no_problem.py:43:6:43:8 | ../src/isfile_no_problem.py:43 | ERROR, you should add `# $ MISSING: flow` annotation | foo |
  failures
 +| ../src/isfile_no_problem.py:43:11:43:41 | Comment # $ flow="SOURCE, l:-15 -> foo" | Missing result:flow="SOURCE, l:-15 -> foo" |

python/ql/test/experimental/dataflow/strange-pointsto-interaction-investigation/test-1-normal/NormalDataflowTest.expected

@@ -1,2 +1,2 @@
-missingAnnotationOnSINK
+missingAnnotationOnSink
 failures

python/ql/test/experimental/dataflow/strange-pointsto-interaction-investigation/test-2-without-splitting/NormalDataflowTest.expected

@@ -1,3 +1,3 @@
-missingAnnotationOnSINK
+missingAnnotationOnSink
 failures
 | ../src/urandom_problem.py:43:6:43:8 | ControlFlowNode for foo | Fixed missing result:flow="SOURCE, l:-15 -> foo" |

python/ql/test/experimental/dataflow/strange-pointsto-interaction-investigation/test-3-max-import-depth-0/NormalDataflowTest.expected

@@ -1,3 +1,3 @@
-missingAnnotationOnSINK
+missingAnnotationOnSink
 failures
 | ../src/urandom_problem.py:43:6:43:8 | ControlFlowNode for foo | Fixed missing result:flow="SOURCE, l:-15 -> foo" |

python/ql/test/experimental/dataflow/strange-pointsto-interaction-investigation/test-4-max-import-depth-100/NormalDataflowTest.expected

@@ -1,3 +1,3 @@
-missingAnnotationOnSINK
+missingAnnotationOnSink
 failures
 | ../src/urandom_problem.py:43:6:43:8 | ControlFlowNode for foo | Fixed missing result:flow="SOURCE, l:-15 -> foo" |

python/ql/test/experimental/dataflow/strange-pointsto-interaction-investigation/test-5-max-import-depth-3/NormalDataflowTest.expected

@@ -1,3 +1,3 @@
-missingAnnotationOnSINK
+missingAnnotationOnSink
 failures
 | ../src/urandom_problem.py:43:6:43:8 | ControlFlowNode for foo | Fixed missing result:flow="SOURCE, l:-15 -> foo" |

python/ql/test/experimental/dataflow/strange-pointsto-interaction-investigation/test-6-max-import-depth-2/NormalDataflowTest.expected

@@ -1,4 +1,4 @@
-missingAnnotationOnSINK
+missingAnnotationOnSink
 | ../src/isfile_no_problem.py:43:6:43:8 | ../src/isfile_no_problem.py:43 | ERROR, you should add `# $ MISSING: flow` annotation | foo |
 failures
 | ../src/isfile_no_problem.py:43:11:43:41 | Comment # $ flow="SOURCE, l:-15 -> foo" | Missing result:flow="SOURCE, l:-15 -> foo" |

python/ql/test/experimental/dataflow/typetracking/test.py

@@ -158,3 +158,27 @@ def test_long_import_chain_full_path():
     from foo.bar import baz # $ tracked_foo_bar_baz
     x = baz # $ tracked_foo_bar_baz
     do_stuff(x) # $ tracked_foo_bar_baz
+
+# ------------------------------------------------------------------------------
+# Global variable to method body flow
+# ------------------------------------------------------------------------------
+
+some_value = get_tracked() # $ tracked
+other_value = get_tracked() # $ tracked
+print(some_value) # $ tracked
+print(other_value) # $ tracked
+
+class MyClass(object):
+    # Since we define some_value method on the class, flow for some_value gets blocked
+    # into the methods
+    def some_value(self):
+        print(some_value) # $ tracked
+        print(other_value) # $ tracked
+
+    def other_name(self):
+        print(some_value) # $ tracked
+        print(other_value) # $ tracked
+
+    def with_global_modifier(self):
+        global some_value
+        print(some_value) # $ tracked

python/ql/test/experimental/dataflow/typetracking_imports/README.md

@@ -0,0 +1,5 @@
+A testcase observed in real code, where mixing `from .this import that` with `from .other import *` (in that order) causes import resolution to not work properly.
+
+This needs to be in a separate folder, since using relative imports requires a valid top-level package. We emulate real extractor behavior using `-R` extractor option.
+
+From this directory, you can run the code with `python -m pkg.use`.

python/ql/test/experimental/dataflow/typetracking_imports/highlight_problem.expected

@@ -0,0 +1,9 @@
+| pkg/alias_only_direct.py:0:0:0:0 | Module pkg.alias_only_direct | pkg/alias_only_direct.py:1:22:1:24 | GSSA Variable foo | use to normal exit |
+| pkg/alias_problem.py:0:0:0:0 | Module pkg.alias_problem | pkg/alias_problem.py:1:22:1:24 | GSSA Variable foo | no use to normal exit |
+| pkg/alias_problem.py:0:0:0:0 | Module pkg.alias_problem | pkg/alias_problem.py:2:1:2:20 | GSSA Variable foo | use to normal exit |
+| pkg/alias_problem_fixed.py:0:0:0:0 | Module pkg.alias_problem_fixed | pkg/alias_problem_fixed.py:0:0:0:0 | GSSA Variable foo | no use to normal exit |
+| pkg/alias_problem_fixed.py:0:0:0:0 | Module pkg.alias_problem_fixed | pkg/alias_problem_fixed.py:3:22:3:24 | GSSA Variable foo | use to normal exit |
+| pkg/problem_absolute_import.py:0:0:0:0 | Module pkg.problem_absolute_import | pkg/problem_absolute_import.py:1:25:1:27 | GSSA Variable foo | no use to normal exit |
+| pkg/problem_absolute_import.py:0:0:0:0 | Module pkg.problem_absolute_import | pkg/problem_absolute_import.py:2:1:2:23 | GSSA Variable foo | use to normal exit |
+| pkg/works_absolute_import.py:0:0:0:0 | Module pkg.works_absolute_import | pkg/works_absolute_import.py:0:0:0:0 | GSSA Variable foo | no use to normal exit |
+| pkg/works_absolute_import.py:0:0:0:0 | Module pkg.works_absolute_import | pkg/works_absolute_import.py:2:25:2:27 | GSSA Variable foo | use to normal exit |

python/ql/test/experimental/dataflow/typetracking_imports/highlight_problem.ql

@@ -0,0 +1,15 @@
+import python
+
+// looking at `module_export` predicate in DataFlowPrivate, the core of the problem is
+// that in alias_problem.py, the direct import of `foo` does not flow to a normal exit of
+// the module. Instead there is a second variable foo coming from `from .other import*` that
+// goes to the normal exit of the module.
+from Module m, EssaVariable v, string useToNormalExit
+where
+  m = v.getScope().getEnclosingModule() and
+  not m.getName() in ["pkg.use", "pkg.foo_def"] and
+  v.getName() = "foo" and
+  if v.getAUse() = m.getANormalExit()
+  then useToNormalExit = "use to normal exit"
+  else useToNormalExit = "no use to normal exit"
+select m, v, useToNormalExit

python/ql/test/experimental/dataflow/typetracking_imports/options

@@ -0,0 +1 @@
+semmle-extractor-options: --max-import-depth=1 -R ./pkg/

python/ql/test/experimental/dataflow/typetracking_imports/pkg/alias_only_direct.py

@@ -0,0 +1 @@
+from .foo_def import foo # $ tracked

python/ql/test/experimental/dataflow/typetracking_imports/pkg/alias_problem.py

@@ -0,0 +1,2 @@
+from .foo_def import foo # $ tracked
+from .other import *

python/ql/test/experimental/dataflow/typetracking_imports/pkg/alias_problem_fixed.py

@@ -0,0 +1,3 @@
+# this ordering makes the problem go away
+from .other import *
+from .foo_def import foo # $ tracked

python/ql/test/experimental/dataflow/typetracking_imports/pkg/alias_star.py

@@ -0,0 +1,2 @@
+from .foo_def import *
+from .other import *

python/ql/test/experimental/dataflow/typetracking_imports/pkg/foo_def.py

@@ -0,0 +1,5 @@
+# apparently adding the assignment makes type-tracker unhappy, so we add this eval so
+# it's possible to run the example and see that everything works
+exec("tracked = 'tracked'")
+foo = tracked # $ tracked
+print(foo) # $ tracked

python/ql/test/experimental/dataflow/typetracking_imports/pkg/other.py

@@ -0,0 +1 @@
+bar = "bar-text"

python/ql/test/experimental/dataflow/typetracking_imports/pkg/problem_absolute_import.py

@@ -0,0 +1,2 @@
+from pkg.foo_def import foo # $ tracked
+from pkg.other import *

python/ql/test/experimental/dataflow/typetracking_imports/pkg/use.py

@@ -0,0 +1,46 @@
+def test_direct_import():
+    from .foo_def import foo # $ tracked
+    print(foo) # $ tracked
+
+test_direct_import()
+
+
+def test_alias_problem():
+    from .alias_problem import foo # $ MISSING: tracked
+    print(foo) # $ MISSING: tracked
+
+test_alias_problem()
+
+
+def test_alias_problem_fixed():
+    from .alias_problem_fixed import foo # $ tracked
+    print(foo) # $ tracked
+
+test_alias_problem_fixed()
+
+
+def test_alias_star():
+    from .alias_star import foo # $ tracked
+    print(foo) # $ tracked
+
+test_alias_star()
+
+
+def test_alias_only_direct():
+    from .alias_only_direct import foo # $ tracked
+    print(foo) # $ tracked
+
+test_alias_only_direct()
+
+
+def test_problem_absolute_import():
+    from pkg.problem_absolute_import import foo # $ MISSING: tracked
+    print(foo) # $ MISSING: tracked
+
+test_problem_absolute_import()
+
+def test_works_absolute_import():
+    from pkg.works_absolute_import import foo # $ tracked
+    print(foo) # $ tracked
+
+test_works_absolute_import()

python/ql/test/experimental/dataflow/typetracking_imports/pkg/works_absolute_import.py

@@ -0,0 +1,2 @@
+from pkg.other import *
+from pkg.foo_def import foo # $ tracked

python/ql/test/experimental/dataflow/typetracking_imports/tracked.qlref

@@ -0,0 +1 @@
+../typetracking/tracked.ql

python/ql/test/library-tests/ApiGraphs/py3/test_entry_point.py

@@ -0,0 +1,2 @@
+"magic_string".foo.bar #$ use=entryPoint("CustomEntryPoint").getMember("foo").getMember("bar")
+"magic_string2".foo.bar

python/ql/test/library-tests/ApiGraphs/py3/verifyApiGraphs.ql

@@ -1,3 +1,11 @@
 // Note: This is not using standard inline-expectation tests, so will not alert if you
 // have not manually added an annotation to a line!
 import TestUtilities.VerifyApiGraphs
+
+class CustomEntryPoint extends API::EntryPoint {
+  CustomEntryPoint() { this = "CustomEntryPoint" }
+
+  override DataFlow::LocalSourceNode getASource() {
+    result.asExpr().(StrConst).getText() = "magic_string"
+  }
+}

python/ql/test/library-tests/PointsTo/new/ImpliesDataflow.expected

@@ -12,6 +12,5 @@
 | code/q_super.py:37:14:37:17 | ControlFlowNode for self | code/q_super.py:27:32:27:35 | ControlFlowNode for self |
 | code/q_super.py:48:5:48:17 | ControlFlowNode for ClassExpr | code/q_super.py:51:25:51:29 | ControlFlowNode for Attribute |
 | code/q_super.py:63:5:63:17 | ControlFlowNode for ClassExpr | code/q_super.py:66:19:66:23 | ControlFlowNode for Attribute |
-| code/r_regressions.py:46:1:46:14 | ControlFlowNode for FunctionExpr | code/r_regressions.py:52:9:52:12 | ControlFlowNode for fail |
 | code/t_type.py:3:1:3:16 | ControlFlowNode for ClassExpr | code/t_type.py:6:1:6:9 | ControlFlowNode for type() |
 | code/t_type.py:3:1:3:16 | ControlFlowNode for ClassExpr | code/t_type.py:13:5:13:13 | ControlFlowNode for type() |

python/ql/test/library-tests/frameworks/django-orm/NormalDataflowTest.expected

@@ -1,2 +1,2 @@
-missingAnnotationOnSINK
+missingAnnotationOnSink
 failures

python/ql/test/library-tests/variables/scopes/access.expected

@@ -0,0 +1,124 @@
+| in_class.py:0:0:0:0 | Module in_class | Global Variable MyClass | in_class.py:5:7:5:13 | MyClass |
+| in_class.py:0:0:0:0 | Module in_class | Global Variable MyClass | in_class.py:33:6:33:12 | MyClass |
+| in_class.py:0:0:0:0 | Module in_class | Global Variable MyClass | in_class.py:41:7:41:13 | MyClass |
+| in_class.py:0:0:0:0 | Module in_class | Global Variable NameError | in_class.py:17:16:17:24 | NameError |
+| in_class.py:0:0:0:0 | Module in_class | Global Variable bar | in_class.py:2:1:2:3 | bar |
+| in_class.py:0:0:0:0 | Module in_class | Global Variable bar | in_class.py:14:15:14:17 | bar |
+| in_class.py:0:0:0:0 | Module in_class | Global Variable baz | in_class.py:16:19:16:21 | baz |
+| in_class.py:0:0:0:0 | Module in_class | Global Variable foo | in_class.py:1:1:1:3 | foo |
+| in_class.py:0:0:0:0 | Module in_class | Global Variable foo | in_class.py:13:15:13:17 | foo |
+| in_class.py:0:0:0:0 | Module in_class | Global Variable foo | in_class.py:31:17:31:19 | foo |
+| in_class.py:0:0:0:0 | Module in_class | Global Variable mc | in_class.py:33:1:33:2 | mc |
+| in_class.py:0:0:0:0 | Module in_class | Global Variable mc | in_class.py:35:1:35:2 | mc |
+| in_class.py:0:0:0:0 | Module in_class | Global Variable mc | in_class.py:38:7:38:8 | mc |
+| in_class.py:0:0:0:0 | Module in_class | Global Variable object | in_class.py:5:15:5:20 | object |
+| in_class.py:0:0:0:0 | Module in_class | Global Variable object | in_class.py:30:15:30:20 | object |
+| in_class.py:0:0:0:0 | Module in_class | Global Variable print | in_class.py:9:9:9:13 | print |
+| in_class.py:0:0:0:0 | Module in_class | Global Variable print | in_class.py:12:9:12:13 | print |
+| in_class.py:0:0:0:0 | Module in_class | Global Variable print | in_class.py:13:9:13:13 | print |
+| in_class.py:0:0:0:0 | Module in_class | Global Variable print | in_class.py:14:9:14:13 | print |
+| in_class.py:0:0:0:0 | Module in_class | Global Variable print | in_class.py:16:13:16:17 | print |
+| in_class.py:0:0:0:0 | Module in_class | Global Variable print | in_class.py:20:9:20:13 | print |
+| in_class.py:0:0:0:0 | Module in_class | Global Variable print | in_class.py:21:9:21:13 | print |
+| in_class.py:0:0:0:0 | Module in_class | Global Variable print | in_class.py:22:9:22:13 | print |
+| in_class.py:0:0:0:0 | Module in_class | Global Variable print | in_class.py:25:9:25:13 | print |
+| in_class.py:0:0:0:0 | Module in_class | Global Variable print | in_class.py:34:1:34:5 | print |
+| in_class.py:0:0:0:0 | Module in_class | Global Variable print | in_class.py:37:1:37:5 | print |
+| in_class.py:0:0:0:0 | Module in_class | Global Variable print | in_class.py:38:1:38:5 | print |
+| in_class.py:0:0:0:0 | Module in_class | Global Variable print | in_class.py:40:1:40:5 | print |
+| in_class.py:0:0:0:0 | Module in_class | Global Variable print | in_class.py:41:1:41:5 | print |
+| in_class.py:5:1:5:22 | Class MyClass | Local Variable Sub | in_class.py:30:11:30:13 | Sub |
+| in_class.py:5:1:5:22 | Class MyClass | Local Variable baz | in_class.py:6:5:6:7 | baz |
+| in_class.py:5:1:5:22 | Class MyClass | Local Variable baz | in_class.py:28:15:28:17 | baz |
+| in_class.py:5:1:5:22 | Class MyClass | Local Variable ex | in_class.py:28:5:28:6 | ex |
+| in_class.py:5:1:5:22 | Class MyClass | Local Variable foo | in_class.py:8:9:8:11 | foo |
+| in_class.py:5:1:5:22 | Class MyClass | Local Variable func | in_class.py:24:9:24:12 | func |
+| in_class.py:5:1:5:22 | Class MyClass | Local Variable func | in_class.py:28:10:28:13 | func |
+| in_class.py:5:1:5:22 | Class MyClass | Local Variable use | in_class.py:11:9:11:11 | use |
+| in_class.py:8:5:8:18 | Function foo | Local Variable self | in_class.py:8:13:8:16 | self |
+| in_class.py:11:5:11:18 | Function use | Local Variable self | in_class.py:11:13:11:16 | self |
+| in_class.py:11:5:11:18 | Function use | Local Variable self | in_class.py:21:15:21:18 | self |
+| in_class.py:11:5:11:18 | Function use | Local Variable self | in_class.py:22:15:22:18 | self |
+| in_class.py:24:5:24:18 | Function func | Local Variable arg | in_class.py:24:14:24:16 | arg |
+| in_class.py:24:5:24:18 | Function func | Local Variable arg | in_class.py:25:27:25:29 | arg |
+| in_class.py:30:5:30:22 | Class Sub | Local Variable stuff | in_class.py:31:9:31:13 | stuff |
+| test.py:0:0:0:0 | Module test | Global Variable C | test.py:30:7:30:7 | C |
+| test.py:0:0:0:0 | Module test | Global Variable base | test.py:30:9:30:12 | base |
+| test.py:0:0:0:0 | Module test | Global Variable func0 | test.py:5:5:5:9 | func0 |
+| test.py:0:0:0:0 | Module test | Global Variable func1 | test.py:8:5:8:9 | func1 |
+| test.py:0:0:0:0 | Module test | Global Variable func2 | test.py:15:5:15:9 | func2 |
+| test.py:0:0:0:0 | Module test | Global Variable func3 | test.py:22:5:22:9 | func3 |
+| test.py:0:0:0:0 | Module test | Global Variable func4 | test.py:38:5:38:9 | func4 |
+| test.py:0:0:0:0 | Module test | Global Variable func5 | test.py:44:5:44:9 | func5 |
+| test.py:0:0:0:0 | Module test | Global Variable func6 | test.py:47:5:47:9 | func6 |
+| test.py:0:0:0:0 | Module test | Global Variable global0 | test.py:2:1:2:7 | global0 |
+| test.py:0:0:0:0 | Module test | Global Variable global0 | test.py:13:5:13:11 | global0 |
+| test.py:0:0:0:0 | Module test | Global Variable global1 | test.py:3:1:3:7 | global1 |
+| test.py:0:0:0:0 | Module test | Global Variable global1 | test.py:13:33:13:39 | global1 |
+| test.py:0:0:0:0 | Module test | Global Variable global_local | test.py:12:5:12:16 | global_local |
+| test.py:0:0:0:0 | Module test | Global Variable range | test.py:52:17:52:21 | range |
+| test.py:0:0:0:0 | Module test | Global Variable seq | test.py:48:26:48:28 | seq |
+| test.py:0:0:0:0 | Module test | Global Variable use_in_loop | test.py:51:5:51:15 | use_in_loop |
+| test.py:5:1:5:26 | Function func0 | Local Variable param0 | test.py:5:11:5:16 | param0 |
+| test.py:5:1:5:26 | Function func0 | Local Variable param0 | test.py:6:12:6:17 | param0 |
+| test.py:5:1:5:26 | Function func0 | Local Variable param1 | test.py:5:19:5:24 | param1 |
+| test.py:5:1:5:26 | Function func0 | Local Variable param1 | test.py:6:21:6:26 | param1 |
+| test.py:8:1:8:12 | Function func1 | Local Variable local0 | test.py:10:5:10:10 | local0 |
+| test.py:8:1:8:12 | Function func1 | Local Variable local0 | test.py:13:15:13:20 | local0 |
+| test.py:8:1:8:12 | Function func1 | Local Variable local1 | test.py:11:5:11:10 | local1 |
+| test.py:8:1:8:12 | Function func1 | Local Variable local1 | test.py:13:24:13:29 | local1 |
+| test.py:15:1:15:12 | Function func2 | Local Variable inner1 | test.py:17:9:17:14 | inner1 |
+| test.py:15:1:15:12 | Function func2 | Local Variable inner1 | test.py:20:12:20:17 | inner1 |
+| test.py:15:1:15:12 | Function func2 | Local Variable local2 | test.py:16:5:16:10 | local2 |
+| test.py:15:1:15:12 | Function func2 | Local Variable local2 | test.py:18:18:18:23 | local2 |
+| test.py:17:5:17:23 | Function inner1 | Local Variable local3 | test.py:18:9:18:14 | local3 |
+| test.py:17:5:17:23 | Function inner1 | Local Variable local3 | test.py:19:16:19:21 | local3 |
+| test.py:17:5:17:23 | Function inner1 | Local Variable param2 | test.py:17:16:17:21 | param2 |
+| test.py:22:1:22:26 | Function func3 | Local Variable inner_outer | test.py:24:9:24:19 | inner_outer |
+| test.py:22:1:22:26 | Function func3 | Local Variable local4 | test.py:23:5:23:10 | local4 |
+| test.py:22:1:22:26 | Function func3 | Local Variable local4 | test.py:26:29:26:34 | local4 |
+| test.py:22:1:22:26 | Function func3 | Local Variable local4 | test.py:28:23:28:28 | local4 |
+| test.py:22:1:22:26 | Function func3 | Local Variable param4 | test.py:22:11:22:16 | param4 |
+| test.py:22:1:22:26 | Function func3 | Local Variable param4 | test.py:26:47:26:52 | param4 |
+| test.py:22:1:22:26 | Function func3 | Local Variable param4 | test.py:28:32:28:37 | param4 |
+| test.py:22:1:22:26 | Function func3 | Local Variable param5 | test.py:22:19:22:24 | param5 |
+| test.py:22:1:22:26 | Function func3 | Local Variable param5 | test.py:28:41:28:46 | param5 |
+| test.py:24:5:24:22 | Function inner_outer | Local Variable inner2 | test.py:25:13:25:18 | inner2 |
+| test.py:24:5:24:22 | Function inner_outer | Local Variable inner2 | test.py:28:16:28:21 | inner2 |
+| test.py:24:5:24:22 | Function inner_outer | Local Variable local5 | test.py:26:20:26:25 | local5 |
+| test.py:24:5:24:22 | Function inner_outer | Local Variable local5 | test.py:27:9:27:14 | local5 |
+| test.py:25:9:25:27 | Function inner2 | Local Variable param3 | test.py:25:20:25:25 | param3 |
+| test.py:25:9:25:27 | Function inner2 | Local Variable param3 | test.py:26:38:26:43 | param3 |
+| test.py:30:1:30:14 | Class C | Local Variable class_local | test.py:32:5:32:15 | class_local |
+| test.py:30:1:30:14 | Class C | Local Variable meth | test.py:34:9:34:12 | meth |
+| test.py:34:5:34:19 | Function meth | Local Variable mlocal | test.py:35:9:35:14 | mlocal |
+| test.py:34:5:34:19 | Function meth | Local Variable mlocal | test.py:36:16:36:21 | mlocal |
+| test.py:34:5:34:19 | Function meth | Local Variable self | test.py:34:14:34:17 | self |
+| test.py:34:5:34:19 | Function meth | Local Variable self | test.py:35:18:35:21 | self |
+| test.py:38:1:38:18 | Function func4 | Local Variable Local | test.py:39:11:39:15 | Local |
+| test.py:38:1:38:18 | Function func4 | Local Variable Local | test.py:42:12:42:16 | Local |
+| test.py:38:1:38:18 | Function func4 | Local Variable param6 | test.py:38:11:38:16 | param6 |
+| test.py:38:1:38:18 | Function func4 | Local Variable param6 | test.py:41:20:41:25 | param6 |
+| test.py:39:5:39:16 | Class Local | Local Variable meth_inner | test.py:40:13:40:22 | meth_inner |
+| test.py:40:9:40:29 | Function meth_inner | Local Variable self | test.py:40:24:40:27 | self |
+| test.py:44:1:44:15 | Function func5 | Local Variable seq | test.py:44:11:44:13 | seq |
+| test.py:44:1:44:15 | Function func5 | Local Variable seq | test.py:45:24:45:26 | seq |
+| test.py:45:12:45:27 | Function listcomp | Local Variable .0 | test.py:45:12:45:27 | .0 |
+| test.py:45:12:45:27 | Function listcomp | Local Variable .0 | test.py:45:12:45:27 | .0 |
+| test.py:45:12:45:27 | Function listcomp | Local Variable x | test.py:45:13:45:13 | x |
+| test.py:45:12:45:27 | Function listcomp | Local Variable x | test.py:45:19:45:19 | x |
+| test.py:47:1:47:16 | Function func6 | Local Variable y | test.py:47:11:47:11 | y |
+| test.py:47:1:47:16 | Function func6 | Local Variable z | test.py:47:14:47:14 | z |
+| test.py:47:1:47:16 | Function func6 | Local Variable z | test.py:48:15:48:15 | z |
+| test.py:48:12:48:29 | Function listcomp | Local Variable .0 | test.py:48:12:48:29 | .0 |
+| test.py:48:12:48:29 | Function listcomp | Local Variable .0 | test.py:48:12:48:29 | .0 |
+| test.py:48:12:48:29 | Function listcomp | Local Variable y | test.py:48:13:48:13 | y |
+| test.py:48:12:48:29 | Function listcomp | Local Variable y | test.py:48:21:48:21 | y |
+| test.py:51:1:51:21 | Function use_in_loop | Local Variable seq | test.py:51:17:51:19 | seq |
+| test.py:51:1:51:21 | Function use_in_loop | Local Variable seq | test.py:53:14:53:16 | seq |
+| test.py:51:1:51:21 | Function use_in_loop | Local Variable v | test.py:53:9:53:9 | v |
+| test.py:51:1:51:21 | Function use_in_loop | Local Variable v | test.py:54:9:54:9 | v |
+| test.py:52:5:52:25 | Function listcomp | Local Variable .0 | test.py:52:5:52:25 | .0 |
+| test.py:52:5:52:25 | Function listcomp | Local Variable .0 | test.py:52:5:52:25 | .0 |
+| test.py:52:5:52:25 | Function listcomp | Local Variable v | test.py:52:6:52:6 | v |
+| test.py:52:5:52:25 | Function listcomp | Local Variable v | test.py:52:12:52:12 | v |

python/ql/test/library-tests/variables/scopes/access.ql

@@ -0,0 +1,7 @@
+import python
+
+from Variable v, Scope s, Name n
+where
+  n = v.getAnAccess() and
+  s = v.getScope()
+select s, v, n

python/ql/test/library-tests/variables/scopes/free.expected

@@ -1,9 +1,9 @@
-| Local Variable local2 | Function func2 | Function inner1 |
-| Local Variable local4 | Function func3 | Function inner2 |
-| Local Variable local4 | Function func3 | Function inner_outer |
-| Local Variable local5 | Function inner_outer | Function inner2 |
-| Local Variable param4 | Function func3 | Function inner2 |
-| Local Variable param4 | Function func3 | Function inner_outer |
-| Local Variable param5 | Function func3 | Function inner_outer |
-| Local Variable param6 | Function func4 | Function meth_inner |
-| Local Variable z | Function func6 | Function listcomp |
+| Local Variable local2 | test.py:15:1:15:12 | Function func2 | test.py:17:5:17:23 | Function inner1 |
+| Local Variable local4 | test.py:22:1:22:26 | Function func3 | test.py:24:5:24:22 | Function inner_outer |
+| Local Variable local4 | test.py:22:1:22:26 | Function func3 | test.py:25:9:25:27 | Function inner2 |
+| Local Variable local5 | test.py:24:5:24:22 | Function inner_outer | test.py:25:9:25:27 | Function inner2 |
+| Local Variable param4 | test.py:22:1:22:26 | Function func3 | test.py:24:5:24:22 | Function inner_outer |
+| Local Variable param4 | test.py:22:1:22:26 | Function func3 | test.py:25:9:25:27 | Function inner2 |
+| Local Variable param5 | test.py:22:1:22:26 | Function func3 | test.py:24:5:24:22 | Function inner_outer |
+| Local Variable param6 | test.py:38:1:38:18 | Function func4 | test.py:40:9:40:29 | Function meth_inner |
+| Local Variable z | test.py:47:1:47:16 | Function func6 | test.py:48:12:48:29 | Function listcomp |

python/ql/test/library-tests/variables/scopes/free.ql

@@ -5,4 +5,4 @@ where
   v.escapes() and
   inner = v.getAnAccess().getScope() and
   inner != v.getScope()
-select v.toString(), v.getScope().toString(), inner.toString()
+select v, v.getScope(), inner

python/ql/test/library-tests/variables/scopes/globals.expected

@@ -1,17 +1,27 @@
-| Global Variable C | Module test |
-| Global Variable __name__ | Module test |
-| Global Variable __package__ | Module test |
-| Global Variable base | Module test |
-| Global Variable func0 | Module test |
-| Global Variable func1 | Module test |
-| Global Variable func2 | Module test |
-| Global Variable func3 | Module test |
-| Global Variable func4 | Module test |
-| Global Variable func5 | Module test |
-| Global Variable func6 | Module test |
-| Global Variable global0 | Module test |
-| Global Variable global1 | Module test |
-| Global Variable global_local | Module test |
-| Global Variable range | Module test |
-| Global Variable seq | Module test |
-| Global Variable use_in_loop | Module test |
+| Global Variable C | test.py:0:0:0:0 | Module test |
+| Global Variable MyClass | in_class.py:0:0:0:0 | Module in_class |
+| Global Variable NameError | in_class.py:0:0:0:0 | Module in_class |
+| Global Variable __name__ | in_class.py:0:0:0:0 | Module in_class |
+| Global Variable __name__ | test.py:0:0:0:0 | Module test |
+| Global Variable __package__ | in_class.py:0:0:0:0 | Module in_class |
+| Global Variable __package__ | test.py:0:0:0:0 | Module test |
+| Global Variable bar | in_class.py:0:0:0:0 | Module in_class |
+| Global Variable base | test.py:0:0:0:0 | Module test |
+| Global Variable baz | in_class.py:0:0:0:0 | Module in_class |
+| Global Variable foo | in_class.py:0:0:0:0 | Module in_class |
+| Global Variable func0 | test.py:0:0:0:0 | Module test |
+| Global Variable func1 | test.py:0:0:0:0 | Module test |
+| Global Variable func2 | test.py:0:0:0:0 | Module test |
+| Global Variable func3 | test.py:0:0:0:0 | Module test |
+| Global Variable func4 | test.py:0:0:0:0 | Module test |
+| Global Variable func5 | test.py:0:0:0:0 | Module test |
+| Global Variable func6 | test.py:0:0:0:0 | Module test |
+| Global Variable global0 | test.py:0:0:0:0 | Module test |
+| Global Variable global1 | test.py:0:0:0:0 | Module test |
+| Global Variable global_local | test.py:0:0:0:0 | Module test |
+| Global Variable mc | in_class.py:0:0:0:0 | Module in_class |
+| Global Variable object | in_class.py:0:0:0:0 | Module in_class |
+| Global Variable print | in_class.py:0:0:0:0 | Module in_class |
+| Global Variable range | test.py:0:0:0:0 | Module test |
+| Global Variable seq | test.py:0:0:0:0 | Module test |
+| Global Variable use_in_loop | test.py:0:0:0:0 | Module test |

python/ql/test/library-tests/variables/scopes/globals.ql

@@ -1,4 +1,4 @@
 import python
 
 from GlobalVariable l
-select l.toString(), l.getScope().toString()
+select l, l.getScope()