Python: Minimal compilation of shared dataflow

python/ql/src/semmle/code/python/dataflow/DataFlow.qll

@@ -0,0 +1,10 @@
+/**
+ * Provides classes for performing local (intra-procedural) and
+ * global (inter-procedural) data flow analyses.
+ */
+
+import python
+
+module DataFlow {
+  import semmle.code.python.dataflow.internal.DataFlowImpl
+}

python/ql/src/semmle/code/python/dataflow/DataFlow2.qll

@@ -0,0 +1,10 @@
+/**
+ * Provides classes for performing local (intra-procedural) and
+ * global (inter-procedural) data flow analyses.
+ */
+
+import python
+
+module DataFlow2 {
+  import semmle.code.python.dataflow.internal.DataFlowImpl2
+}

python/ql/src/semmle/code/python/dataflow/TaintTracking.qll

@@ -0,0 +1,10 @@
+/**
+ * Provides classes for performing local (intra-procedural) and
+ * global (inter-procedural) taint-tracking analyses.
+ */
+
+import python
+
+module TaintTracking {
+  import semmle.code.python.dataflow.internal.tainttracking1.TaintTrackingImpl
+}

python/ql/src/semmle/code/python/dataflow/internal/DataFlowImplCommon.qll

@@ -0,0 +1,735 @@
+private import DataFlowImplSpecific::Private
+private import DataFlowImplSpecific::Public
+import Cached
+
+cached
+private module Cached {
+  /**
+   * Holds if `p` is the `i`th parameter of a viable dispatch target of `call`.
+   * The instance parameter is considered to have index `-1`.
+   */
+  pragma[nomagic]
+  private predicate viableParam(DataFlowCall call, int i, ParameterNode p) {
+    p.isParameterOf(viableCallable(call), i)
+  }
+
+  /**
+   * Holds if `arg` is a possible argument to `p` in `call`, taking virtual
+   * dispatch into account.
+   */
+  cached
+  predicate viableParamArg(DataFlowCall call, ParameterNode p, ArgumentNode arg) {
+    exists(int i |
+      viableParam(call, i, p) and
+      arg.argumentOf(call, i) and
+      compatibleTypes(getErasedNodeTypeBound(arg), getErasedNodeTypeBound(p))
+    )
+  }
+
+  pragma[nomagic]
+  private ReturnPosition viableReturnPos(DataFlowCall call, ReturnKindExt kind) {
+    viableCallable(call) = result.getCallable() and
+    kind = result.getKind()
+  }
+
+  /**
+   * Holds if a value at return position `pos` can be returned to `out` via `call`,
+   * taking virtual dispatch into account.
+   */
+  cached
+  predicate viableReturnPosOut(DataFlowCall call, ReturnPosition pos, Node out) {
+    exists(ReturnKindExt kind |
+      pos = viableReturnPos(call, kind) and
+      out = kind.getAnOutNode(call)
+    )
+  }
+
+  /** Provides predicates for calculating flow-through summaries. */
+  private module FlowThrough {
+    /**
+     * The first flow-through approximation:
+     *
+     * - Input access paths are abstracted with a Boolean parameter
+     *   that indicates (non-)emptiness.
+     */
+    private module Cand {
+      /**
+       * Holds if `p` can flow to `node` in the same callable using only
+       * value-preserving steps.
+       *
+       * `read` indicates whether it is contents of `p` that can flow to `node`.
+       */
+      pragma[nomagic]
+      private predicate parameterValueFlowCand(ParameterNode p, Node node, boolean read) {
+        p = node and
+        read = false
+        or
+        // local flow
+        exists(Node mid |
+          parameterValueFlowCand(p, mid, read) and
+          simpleLocalFlowStep(mid, node)
+        )
+        or
+        // read
+        exists(Node mid |
+          parameterValueFlowCand(p, mid, false) and
+          readStep(mid, _, node) and
+          read = true
+        )
+        or
+        // flow through: no prior read
+        exists(ArgumentNode arg |
+          parameterValueFlowArgCand(p, arg, false) and
+          argumentValueFlowsThroughCand(arg, node, read)
+        )
+        or
+        // flow through: no read inside method
+        exists(ArgumentNode arg |
+          parameterValueFlowArgCand(p, arg, read) and
+          argumentValueFlowsThroughCand(arg, node, false)
+        )
+      }
+
+      pragma[nomagic]
+      private predicate parameterValueFlowArgCand(ParameterNode p, ArgumentNode arg, boolean read) {
+        parameterValueFlowCand(p, arg, read)
+      }
+
+      pragma[nomagic]
+      predicate parameterValueFlowsToPreUpdateCand(ParameterNode p, PostUpdateNode n) {
+        parameterValueFlowCand(p, n.getPreUpdateNode(), false)
+      }
+
+      /**
+       * Holds if `p` can flow to a return node of kind `kind` in the same
+       * callable using only value-preserving steps, not taking call contexts
+       * into account.
+       *
+       * `read` indicates whether it is contents of `p` that can flow to the return
+       * node.
+       */
+      predicate parameterValueFlowReturnCand(ParameterNode p, ReturnKind kind, boolean read) {
+        exists(ReturnNode ret |
+          parameterValueFlowCand(p, ret, read) and
+          kind = ret.getKind()
+        )
+      }
+
+      pragma[nomagic]
+      private predicate argumentValueFlowsThroughCand0(
+        DataFlowCall call, ArgumentNode arg, ReturnKind kind, boolean read
+      ) {
+        exists(ParameterNode param | viableParamArg(call, param, arg) |
+          parameterValueFlowReturnCand(param, kind, read)
+        )
+      }
+
+      /**
+       * Holds if `arg` flows to `out` through a call using only value-preserving steps,
+       * not taking call contexts into account.
+       *
+       * `read` indicates whether it is contents of `arg` that can flow to `out`.
+       */
+      predicate argumentValueFlowsThroughCand(ArgumentNode arg, Node out, boolean read) {
+        exists(DataFlowCall call, ReturnKind kind |
+          argumentValueFlowsThroughCand0(call, arg, kind, read) and
+          out = getAnOutNode(call, kind)
+        )
+      }
+
+      predicate cand(ParameterNode p, Node n) {
+        parameterValueFlowCand(p, n, _) and
+        (
+          parameterValueFlowReturnCand(p, _, _)
+          or
+          parameterValueFlowsToPreUpdateCand(p, _)
+        )
+      }
+    }
+
+    private module LocalFlowBigStep {
+      private predicate localFlowEntry(Node n) {
+        Cand::cand(_, n) and
+        (
+          n instanceof ParameterNode or
+          n instanceof OutNode or
+          readStep(_, _, n) or
+          n instanceof CastNode
+        )
+      }
+
+      private predicate localFlowExit(Node n) {
+        Cand::cand(_, n) and
+        (
+          n instanceof ArgumentNode
+          or
+          n instanceof ReturnNode
+          or
+          readStep(n, _, _)
+          or
+          n instanceof CastNode
+          or
+          n =
+            any(PostUpdateNode pun | Cand::parameterValueFlowsToPreUpdateCand(_, pun))
+                .getPreUpdateNode()
+        )
+      }
+
+      pragma[nomagic]
+      private predicate localFlowStepPlus(Node node1, Node node2) {
+        localFlowEntry(node1) and
+        simpleLocalFlowStep(node1, node2) and
+        node1 != node2
+        or
+        exists(Node mid |
+          localFlowStepPlus(node1, mid) and
+          simpleLocalFlowStep(mid, node2) and
+          not mid instanceof CastNode
+        )
+      }
+
+      pragma[nomagic]
+      predicate localFlowBigStep(Node node1, Node node2) {
+        localFlowStepPlus(node1, node2) and
+        localFlowExit(node2)
+      }
+    }
+
+    /**
+     * The final flow-through calculation:
+     *
+     * - Input access paths are abstracted with a `ContentOption` parameter
+     *   that represents the head of the access path. `TContentNone()` means that
+     *   the access path is unrestricted.
+     * - Types are checked using the `compatibleTypes()` relation.
+     */
+    private module Final {
+      /**
+       * Holds if `p` can flow to `node` in the same callable using only
+       * value-preserving steps, not taking call contexts into account.
+       *
+       * `contentIn` describes the content of `p` that can flow to `node`
+       * (if any).
+       */
+      predicate parameterValueFlow(ParameterNode p, Node node, ContentOption contentIn) {
+        parameterValueFlow0(p, node, contentIn) and
+        if node instanceof CastingNode
+        then
+          // normal flow through
+          contentIn = TContentNone() and
+          compatibleTypes(getErasedNodeTypeBound(p), getErasedNodeTypeBound(node))
+          or
+          // getter
+          exists(Content fIn |
+            contentIn.getContent() = fIn and
+            compatibleTypes(fIn.getType(), getErasedNodeTypeBound(node))
+          )
+        else any()
+      }
+
+      pragma[nomagic]
+      private predicate parameterValueFlow0(ParameterNode p, Node node, ContentOption contentIn) {
+        p = node and
+        Cand::cand(p, _) and
+        contentIn = TContentNone()
+        or
+        // local flow
+        exists(Node mid |
+          parameterValueFlow(p, mid, contentIn) and
+          LocalFlowBigStep::localFlowBigStep(mid, node)
+        )
+        or
+        // read
+        exists(Node mid, Content f |
+          parameterValueFlow(p, mid, TContentNone()) and
+          readStep(mid, f, node) and
+          contentIn.getContent() = f and
+          Cand::parameterValueFlowReturnCand(p, _, true) and
+          compatibleTypes(getErasedNodeTypeBound(p), f.getContainerType())
+        )
+        or
+        // flow through: no prior read
+        exists(ArgumentNode arg |
+          parameterValueFlowArg(p, arg, TContentNone()) and
+          argumentValueFlowsThrough(arg, contentIn, node)
+        )
+        or
+        // flow through: no read inside method
+        exists(ArgumentNode arg |
+          parameterValueFlowArg(p, arg, contentIn) and
+          argumentValueFlowsThrough(arg, TContentNone(), node)
+        )
+      }
+
+      pragma[nomagic]
+      private predicate parameterValueFlowArg(
+        ParameterNode p, ArgumentNode arg, ContentOption contentIn
+      ) {
+        parameterValueFlow(p, arg, contentIn) and
+        Cand::argumentValueFlowsThroughCand(arg, _, _)
+      }
+
+      pragma[nomagic]
+      private predicate argumentValueFlowsThrough0(
+        DataFlowCall call, ArgumentNode arg, ReturnKind kind, ContentOption contentIn
+      ) {
+        exists(ParameterNode param | viableParamArg(call, param, arg) |
+          parameterValueFlowReturn(param, kind, contentIn)
+        )
+      }
+
+      /**
+       * Holds if `arg` flows to `out` through a call using only value-preserving steps,
+       * not taking call contexts into account.
+       *
+       * `contentIn` describes the content of `arg` that can flow to `out` (if any).
+       */
+      pragma[nomagic]
+      predicate argumentValueFlowsThrough(ArgumentNode arg, ContentOption contentIn, Node out) {
+        exists(DataFlowCall call, ReturnKind kind |
+          argumentValueFlowsThrough0(call, arg, kind, contentIn) and
+          out = getAnOutNode(call, kind)
+        |
+          // normal flow through
+          contentIn = TContentNone() and
+          compatibleTypes(getErasedNodeTypeBound(arg), getErasedNodeTypeBound(out))
+          or
+          // getter
+          exists(Content fIn |
+            contentIn.getContent() = fIn and
+            compatibleTypes(getErasedNodeTypeBound(arg), fIn.getContainerType()) and
+            compatibleTypes(fIn.getType(), getErasedNodeTypeBound(out))
+          )
+        )
+      }
+
+      /**
+       * Holds if `p` can flow to a return node of kind `kind` in the same
+       * callable using only value-preserving steps.
+       *
+       * `contentIn` describes the content of `p` that can flow to the return
+       * node (if any).
+       */
+      private predicate parameterValueFlowReturn(
+        ParameterNode p, ReturnKind kind, ContentOption contentIn
+      ) {
+        exists(ReturnNode ret |
+          parameterValueFlow(p, ret, contentIn) and
+          kind = ret.getKind()
+        )
+      }
+    }
+
+    import Final
+  }
+
+  /**
+   * Holds if `p` can flow to the pre-update node associated with post-update
+   * node `n`, in the same callable, using only value-preserving steps.
+   */
+  cached
+  predicate parameterValueFlowsToPreUpdate(ParameterNode p, PostUpdateNode n) {
+    parameterValueFlow(p, n.getPreUpdateNode(), TContentNone())
+  }
+
+  /**
+   * Holds if data can flow from `node1` to `node2` via a direct assignment to
+   * `f`.
+   *
+   * This includes reverse steps through reads when the result of the read has
+   * been stored into, in order to handle cases like `x.f1.f2 = y`.
+   */
+  cached
+  predicate store(Node node1, Content f, Node node2) {
+    storeStep(node1, f, node2) and readStep(_, f, _)
+    or
+    exists(Node n1, Node n2 |
+      n1 = node1.(PostUpdateNode).getPreUpdateNode() and
+      n2 = node2.(PostUpdateNode).getPreUpdateNode()
+    |
+      argumentValueFlowsThrough(n2, TContentSome(f), n1)
+      or
+      readStep(n2, f, n1)
+    )
+  }
+
+  import FlowThrough
+
+  /**
+   * Holds if the call context `call` either improves virtual dispatch in
+   * `callable` or if it allows us to prune unreachable nodes in `callable`.
+   */
+  cached
+  predicate recordDataFlowCallSite(DataFlowCall call, DataFlowCallable callable) {
+    reducedViableImplInCallContext(_, callable, call)
+    or
+    exists(Node n | n.getEnclosingCallable() = callable | isUnreachableInCall(n, call))
+  }
+
+  cached
+  newtype TCallContext =
+    TAnyCallContext() or
+    TSpecificCall(DataFlowCall call) { recordDataFlowCallSite(call, _) } or
+    TSomeCall() or
+    TReturn(DataFlowCallable c, DataFlowCall call) { reducedViableImplInReturn(c, call) }
+
+  cached
+  newtype TReturnPosition =
+    TReturnPosition0(DataFlowCallable c, ReturnKindExt kind) {
+      exists(ReturnNodeExt ret |
+        c = returnNodeGetEnclosingCallable(ret) and
+        kind = ret.getKind()
+      )
+    }
+
+  cached
+  newtype TLocalFlowCallContext =
+    TAnyLocalCall() or
+    TSpecificLocalCall(DataFlowCall call) { isUnreachableInCall(_, call) }
+
+  cached
+  newtype TReturnKindExt =
+    TValueReturn(ReturnKind kind) or
+    TParamUpdate(int pos) { exists(ParameterNode p | p.isParameterOf(_, pos)) }
+
+  cached
+  newtype TBooleanOption =
+    TBooleanNone() or
+    TBooleanSome(boolean b) { b = true or b = false }
+
+  cached
+  newtype TAccessPathFront =
+    TFrontNil(DataFlowType t) or
+    TFrontHead(Content f)
+
+  cached
+  newtype TAccessPathFrontOption =
+    TAccessPathFrontNone() or
+    TAccessPathFrontSome(AccessPathFront apf)
+}
+
+/**
+ * A `Node` at which a cast can occur such that the type should be checked.
+ */
+class CastingNode extends Node {
+  CastingNode() {
+    this instanceof ParameterNode or
+    this instanceof CastNode or
+    this instanceof OutNodeExt
+  }
+}
+
+newtype TContentOption =
+  TContentNone() or
+  TContentSome(Content f)
+
+private class ContentOption extends TContentOption {
+  Content getContent() { this = TContentSome(result) }
+
+  predicate hasContent() { exists(this.getContent()) }
+
+  string toString() {
+    result = this.getContent().toString()
+    or
+    not this.hasContent() and
+    result = "<none>"
+  }
+}
+
+/**
+ * A call context to restrict the targets of virtual dispatch, prune local flow,
+ * and match the call sites of flow into a method with flow out of a method.
+ *
+ * There are four cases:
+ * - `TAnyCallContext()` : No restrictions on method flow.
+ * - `TSpecificCall(DataFlowCall call)` : Flow entered through the
+ *    given `call`. This call improves the set of viable
+ *    dispatch targets for at least one method call in the current callable
+ *    or helps prune unreachable nodes in the current callable.
+ * - `TSomeCall()` : Flow entered through a parameter. The
+ *    originating call does not improve the set of dispatch targets for any
+ *    method call in the current callable and was therefore not recorded.
+ * - `TReturn(Callable c, DataFlowCall call)` : Flow reached `call` from `c` and
+ *    this dispatch target of `call` implies a reduced set of dispatch origins
+ *    to which data may flow if it should reach a `return` statement.
+ */
+abstract class CallContext extends TCallContext {
+  abstract string toString();
+
+  /** Holds if this call context is relevant for `callable`. */
+  abstract predicate relevantFor(DataFlowCallable callable);
+}
+
+class CallContextAny extends CallContext, TAnyCallContext {
+  override string toString() { result = "CcAny" }
+
+  override predicate relevantFor(DataFlowCallable callable) { any() }
+}
+
+abstract class CallContextCall extends CallContext { }
+
+class CallContextSpecificCall extends CallContextCall, TSpecificCall {
+  override string toString() {
+    exists(DataFlowCall call | this = TSpecificCall(call) | result = "CcCall(" + call + ")")
+  }
+
+  override predicate relevantFor(DataFlowCallable callable) {
+    recordDataFlowCallSite(getCall(), callable)
+  }
+
+  DataFlowCall getCall() { this = TSpecificCall(result) }
+}
+
+class CallContextSomeCall extends CallContextCall, TSomeCall {
+  override string toString() { result = "CcSomeCall" }
+
+  override predicate relevantFor(DataFlowCallable callable) {
+    exists(ParameterNode p | p.getEnclosingCallable() = callable)
+  }
+}
+
+class CallContextReturn extends CallContext, TReturn {
+  override string toString() {
+    exists(DataFlowCall call | this = TReturn(_, call) | result = "CcReturn(" + call + ")")
+  }
+
+  override predicate relevantFor(DataFlowCallable callable) {
+    exists(DataFlowCall call | this = TReturn(_, call) and call.getEnclosingCallable() = callable)
+  }
+}
+
+/**
+ * A call context that is relevant for pruning local flow.
+ */
+abstract class LocalCallContext extends TLocalFlowCallContext {
+  abstract string toString();
+
+  /** Holds if this call context is relevant for `callable`. */
+  abstract predicate relevantFor(DataFlowCallable callable);
+}
+
+class LocalCallContextAny extends LocalCallContext, TAnyLocalCall {
+  override string toString() { result = "LocalCcAny" }
+
+  override predicate relevantFor(DataFlowCallable callable) { any() }
+}
+
+class LocalCallContextSpecificCall extends LocalCallContext, TSpecificLocalCall {
+  LocalCallContextSpecificCall() { this = TSpecificLocalCall(call) }
+
+  DataFlowCall call;
+
+  DataFlowCall getCall() { result = call }
+
+  override string toString() { result = "LocalCcCall(" + call + ")" }
+
+  override predicate relevantFor(DataFlowCallable callable) { relevantLocalCCtx(call, callable) }
+}
+
+private predicate relevantLocalCCtx(DataFlowCall call, DataFlowCallable callable) {
+  exists(Node n | n.getEnclosingCallable() = callable and isUnreachableInCall(n, call))
+}
+
+/**
+ * Gets the local call context given the call context and the callable that
+ * the contexts apply to.
+ */
+LocalCallContext getLocalCallContext(CallContext ctx, DataFlowCallable callable) {
+  ctx.relevantFor(callable) and
+  if relevantLocalCCtx(ctx.(CallContextSpecificCall).getCall(), callable)
+  then result.(LocalCallContextSpecificCall).getCall() = ctx.(CallContextSpecificCall).getCall()
+  else result instanceof LocalCallContextAny
+}
+
+/**
+ * A node from which flow can return to the caller. This is either a regular
+ * `ReturnNode` or a `PostUpdateNode` corresponding to the value of a parameter.
+ */
+class ReturnNodeExt extends Node {
+  ReturnNodeExt() {
+    this instanceof ReturnNode or
+    parameterValueFlowsToPreUpdate(_, this)
+  }
+
+  /** Gets the kind of this returned value. */
+  ReturnKindExt getKind() {
+    result = TValueReturn(this.(ReturnNode).getKind())
+    or
+    exists(ParameterNode p, int pos |
+      parameterValueFlowsToPreUpdate(p, this) and
+      p.isParameterOf(_, pos) and
+      result = TParamUpdate(pos)
+    )
+  }
+}
+
+/**
+ * A node to which data can flow from a call. Either an ordinary out node
+ * or a post-update node associated with a call argument.
+ */
+class OutNodeExt extends Node {
+  OutNodeExt() {
+    this instanceof OutNode
+    or
+    this.(PostUpdateNode).getPreUpdateNode() instanceof ArgumentNode
+  }
+}
+
+/**
+ * An extended return kind. A return kind describes how data can be returned
+ * from a callable. This can either be through a returned value or an updated
+ * parameter.
+ */
+abstract class ReturnKindExt extends TReturnKindExt {
+  /** Gets a textual representation of this return kind. */
+  abstract string toString();
+
+  /** Gets a node corresponding to data flow out of `call`. */
+  abstract OutNodeExt getAnOutNode(DataFlowCall call);
+}
+
+class ValueReturnKind extends ReturnKindExt, TValueReturn {
+  private ReturnKind kind;
+
+  ValueReturnKind() { this = TValueReturn(kind) }
+
+  ReturnKind getKind() { result = kind }
+
+  override string toString() { result = kind.toString() }
+
+  override OutNodeExt getAnOutNode(DataFlowCall call) {
+    result = getAnOutNode(call, this.getKind())
+  }
+}
+
+class ParamUpdateReturnKind extends ReturnKindExt, TParamUpdate {
+  private int pos;
+
+  ParamUpdateReturnKind() { this = TParamUpdate(pos) }
+
+  int getPosition() { result = pos }
+
+  override string toString() { result = "param update " + pos }
+
+  override OutNodeExt getAnOutNode(DataFlowCall call) {
+    exists(ArgumentNode arg |
+      result.(PostUpdateNode).getPreUpdateNode() = arg and
+      arg.argumentOf(call, this.getPosition())
+    )
+  }
+}
+
+/** A callable tagged with a relevant return kind. */
+class ReturnPosition extends TReturnPosition0 {
+  private DataFlowCallable c;
+  private ReturnKindExt kind;
+
+  ReturnPosition() { this = TReturnPosition0(c, kind) }
+
+  /** Gets the callable. */
+  DataFlowCallable getCallable() { result = c }
+
+  /** Gets the return kind. */
+  ReturnKindExt getKind() { result = kind }
+
+  /** Gets a textual representation of this return position. */
+  string toString() { result = "[" + kind + "] " + c }
+}
+
+pragma[noinline]
+private DataFlowCallable returnNodeGetEnclosingCallable(ReturnNodeExt ret) {
+  result = ret.getEnclosingCallable()
+}
+
+pragma[noinline]
+private ReturnPosition getReturnPosition0(ReturnNodeExt ret, ReturnKindExt kind) {
+  result.getCallable() = returnNodeGetEnclosingCallable(ret) and
+  kind = result.getKind()
+}
+
+pragma[noinline]
+ReturnPosition getReturnPosition(ReturnNodeExt ret) {
+  result = getReturnPosition0(ret, ret.getKind())
+}
+
+bindingset[cc, callable]
+predicate resolveReturn(CallContext cc, DataFlowCallable callable, DataFlowCall call) {
+  cc instanceof CallContextAny and callable = viableCallable(call)
+  or
+  exists(DataFlowCallable c0, DataFlowCall call0 |
+    call0.getEnclosingCallable() = callable and
+    cc = TReturn(c0, call0) and
+    c0 = prunedViableImplInCallContextReverse(call0, call)
+  )
+}
+
+bindingset[call, cc]
+DataFlowCallable resolveCall(DataFlowCall call, CallContext cc) {
+  exists(DataFlowCall ctx | cc = TSpecificCall(ctx) |
+    if reducedViableImplInCallContext(call, _, ctx)
+    then result = prunedViableImplInCallContext(call, ctx)
+    else result = viableCallable(call)
+  )
+  or
+  result = viableCallable(call) and cc instanceof CallContextSomeCall
+  or
+  result = viableCallable(call) and cc instanceof CallContextAny
+  or
+  result = viableCallable(call) and cc instanceof CallContextReturn
+}
+
+pragma[noinline]
+DataFlowType getErasedNodeTypeBound(Node n) { result = getErasedRepr(n.getTypeBound()) }
+
+predicate read = readStep/3;
+
+/** An optional Boolean value. */
+class BooleanOption extends TBooleanOption {
+  string toString() {
+    this = TBooleanNone() and result = "<none>"
+    or
+    this = TBooleanSome(any(boolean b | result = b.toString()))
+  }
+}
+
+/**
+ * The front of an access path. This is either a head or a nil.
+ */
+abstract class AccessPathFront extends TAccessPathFront {
+  abstract string toString();
+
+  abstract DataFlowType getType();
+
+  abstract boolean toBoolNonEmpty();
+
+  predicate headUsesContent(Content f) { this = TFrontHead(f) }
+}
+
+class AccessPathFrontNil extends AccessPathFront, TFrontNil {
+  override string toString() {
+    exists(DataFlowType t | this = TFrontNil(t) | result = ppReprType(t))
+  }
+
+  override DataFlowType getType() { this = TFrontNil(result) }
+
+  override boolean toBoolNonEmpty() { result = false }
+}
+
+class AccessPathFrontHead extends AccessPathFront, TFrontHead {
+  override string toString() { exists(Content f | this = TFrontHead(f) | result = f.toString()) }
+
+  override DataFlowType getType() {
+    exists(Content head | this = TFrontHead(head) | result = head.getContainerType())
+  }
+
+  override boolean toBoolNonEmpty() { result = true }
+}
+
+/** An optional access path front. */
+class AccessPathFrontOption extends TAccessPathFrontOption {
+  string toString() {
+    this = TAccessPathFrontNone() and result = "<none>"
+    or
+    this = TAccessPathFrontSome(any(AccessPathFront apf | result = apf.toString()))
+  }
+}

python/ql/src/semmle/code/python/dataflow/internal/DataFlowImplConsistency.qll

@@ -0,0 +1,175 @@
+/**
+ * Provides consistency queries for checking invariants in the language-specific
+ * data-flow classes and predicates.
+ */
+
+private import DataFlowImplSpecific::Private
+private import DataFlowImplSpecific::Public
+private import tainttracking1.TaintTrackingParameter::Private
+private import tainttracking1.TaintTrackingParameter::Public
+
+module Consistency {
+  private class RelevantNode extends Node {
+    RelevantNode() {
+      this instanceof ArgumentNode or
+      this instanceof ParameterNode or
+      this instanceof ReturnNode or
+      this = getAnOutNode(_, _) or
+      simpleLocalFlowStep(this, _) or
+      simpleLocalFlowStep(_, this) or
+      jumpStep(this, _) or
+      jumpStep(_, this) or
+      storeStep(this, _, _) or
+      storeStep(_, _, this) or
+      readStep(this, _, _) or
+      readStep(_, _, this) or
+      defaultAdditionalTaintStep(this, _) or
+      defaultAdditionalTaintStep(_, this)
+    }
+  }
+
+  query predicate uniqueEnclosingCallable(Node n, string msg) {
+    exists(int c |
+      n instanceof RelevantNode and
+      c = count(n.getEnclosingCallable()) and
+      c != 1 and
+      msg = "Node should have one enclosing callable but has " + c + "."
+    )
+  }
+
+  query predicate uniqueTypeBound(Node n, string msg) {
+    exists(int c |
+      n instanceof RelevantNode and
+      c = count(n.getTypeBound()) and
+      c != 1 and
+      msg = "Node should have one type bound but has " + c + "."
+    )
+  }
+
+  query predicate uniqueTypeRepr(Node n, string msg) {
+    exists(int c |
+      n instanceof RelevantNode and
+      c = count(getErasedRepr(n.getTypeBound())) and
+      c != 1 and
+      msg = "Node should have one type representation but has " + c + "."
+    )
+  }
+
+  query predicate uniqueNodeLocation(Node n, string msg) {
+    exists(int c |
+      c =
+        count(string filepath, int startline, int startcolumn, int endline, int endcolumn |
+          n.hasLocationInfo(filepath, startline, startcolumn, endline, endcolumn)
+        ) and
+      c != 1 and
+      msg = "Node should have one location but has " + c + "."
+    )
+  }
+
+  query predicate missingLocation(string msg) {
+    exists(int c |
+      c =
+        strictcount(Node n |
+          not exists(string filepath, int startline, int startcolumn, int endline, int endcolumn |
+            n.hasLocationInfo(filepath, startline, startcolumn, endline, endcolumn)
+          )
+        ) and
+      msg = "Nodes without location: " + c
+    )
+  }
+
+  query predicate uniqueNodeToString(Node n, string msg) {
+    exists(int c |
+      c = count(n.toString()) and
+      c != 1 and
+      msg = "Node should have one toString but has " + c + "."
+    )
+  }
+
+  query predicate missingToString(string msg) {
+    exists(int c |
+      c = strictcount(Node n | not exists(n.toString())) and
+      msg = "Nodes without toString: " + c
+    )
+  }
+
+  query predicate parameterCallable(ParameterNode p, string msg) {
+    exists(DataFlowCallable c | p.isParameterOf(c, _) and c != p.getEnclosingCallable()) and
+    msg = "Callable mismatch for parameter."
+  }
+
+  query predicate localFlowIsLocal(Node n1, Node n2, string msg) {
+    simpleLocalFlowStep(n1, n2) and
+    n1.getEnclosingCallable() != n2.getEnclosingCallable() and
+    msg = "Local flow step does not preserve enclosing callable."
+  }
+
+  private DataFlowType typeRepr() { result = getErasedRepr(any(Node n).getTypeBound()) }
+
+  query predicate compatibleTypesReflexive(DataFlowType t, string msg) {
+    t = typeRepr() and
+    not compatibleTypes(t, t) and
+    msg = "Type compatibility predicate is not reflexive."
+  }
+
+  query predicate unreachableNodeCCtx(Node n, DataFlowCall call, string msg) {
+    isUnreachableInCall(n, call) and
+    exists(DataFlowCallable c |
+      c = n.getEnclosingCallable() and
+      not viableCallable(call) = c
+    ) and
+    msg = "Call context for isUnreachableInCall is inconsistent with call graph."
+  }
+
+  query predicate localCallNodes(DataFlowCall call, Node n, string msg) {
+    (
+      n = getAnOutNode(call, _) and
+      msg = "OutNode and call does not share enclosing callable."
+      or
+      n.(ArgumentNode).argumentOf(call, _) and
+      msg = "ArgumentNode and call does not share enclosing callable."
+    ) and
+    n.getEnclosingCallable() != call.getEnclosingCallable()
+  }
+
+  query predicate postIsNotPre(PostUpdateNode n, string msg) {
+    n.getPreUpdateNode() = n and msg = "PostUpdateNode should not equal its pre-update node."
+  }
+
+  query predicate postHasUniquePre(PostUpdateNode n, string msg) {
+    exists(int c |
+      c = count(n.getPreUpdateNode()) and
+      c != 1 and
+      msg = "PostUpdateNode should have one pre-update node but has " + c + "."
+    )
+  }
+
+  query predicate uniquePostUpdate(Node n, string msg) {
+    1 < strictcount(PostUpdateNode post | post.getPreUpdateNode() = n) and
+    msg = "Node has multiple PostUpdateNodes."
+  }
+
+  query predicate postIsInSameCallable(PostUpdateNode n, string msg) {
+    n.getEnclosingCallable() != n.getPreUpdateNode().getEnclosingCallable() and
+    msg = "PostUpdateNode does not share callable with its pre-update node."
+  }
+
+  private predicate hasPost(Node n) { exists(PostUpdateNode post | post.getPreUpdateNode() = n) }
+
+  query predicate reverseRead(Node n, string msg) {
+    exists(Node n2 | readStep(n, _, n2) and hasPost(n2) and not hasPost(n)) and
+    msg = "Origin of readStep is missing a PostUpdateNode."
+  }
+
+  query predicate storeIsPostUpdate(Node n, string msg) {
+    storeStep(_, _, n) and
+    not n instanceof PostUpdateNode and
+    msg = "Store targets should be PostUpdateNodes."
+  }
+
+  query predicate argHasPostUpdate(ArgumentNode n, string msg) {
+    not hasPost(n) and
+    not isImmutableOrUnobservable(n) and
+    msg = "ArgumentNode is missing PostUpdateNode."
+  }
+}

python/ql/src/semmle/code/python/dataflow/internal/DataFlowImplSpecific.qll

@@ -0,0 +1,11 @@
+/**
+ * Provides Python-specific definitions for use in the data flow library.
+ */
+module Private {
+  import DataFlowPrivate
+//   import DataFlowDispatch
+}
+
+module Public {
+  import DataFlowPublic
+}

python/ql/src/semmle/code/python/dataflow/internal/DataFlowPrivate.qll

@@ -0,0 +1,188 @@
+private import python
+private import DataFlowPublic
+
+class DataFlowCall extends Call {
+  /** Gets the enclosing callable of this call. */
+  abstract DataFlowCallable getEnclosingCallable();
+}
+
+/** A data flow node that represents a call argument. */
+abstract class ArgumentNode extends Node {
+  /** Holds if this argument occurs at the given position in the given call. */
+  cached
+  abstract predicate argumentOf(DataFlowCall call, int pos);
+
+  /** Gets the call in which this node is an argument. */
+  final DataFlowCall getCall() { this.argumentOf(result, _) }
+}
+
+
+/**
+ * A node associated with an object after an operation that might have
+ * changed its state.
+ *
+ * This can be either the argument to a callable after the callable returns
+ * (which might have mutated the argument), or the qualifier of a field after
+ * an update to the field.
+ *
+ * Nodes corresponding to AST elements, for example `ExprNode`, usually refer
+ * to the value before the update with the exception of `ObjectCreation`,
+ * which represents the value after the constructor has run.
+ */
+abstract class PostUpdateNode extends Node {
+  /** Gets the node before the state update. */
+  abstract Node getPreUpdateNode();
+}
+
+/**
+ * A return kind. A return kind describes how a value can be returned
+ * from a callable.
+ */
+abstract class ReturnKind extends string {
+  /** Gets a textual representation of this position. */
+  ReturnKind() { this = "ReturnKind" }
+}
+
+/** A data flow node that represents a value returned by a callable. */
+abstract class ReturnNode extends Node {
+  /** Gets the kind of this return node. */
+  abstract ReturnKind getKind();
+}
+
+/** A data flow node that represents the output of a call. */
+abstract class OutNode extends Node {
+  /** Gets the underlying call, where this node is a corresponding output of kind `kind`. */
+  cached
+  abstract DataFlowCall getCall(ReturnKind kind);
+}
+
+/** A node that performs a type cast. */
+class CastNode extends Node {
+}
+
+class DataFlowCallable = FunctionValue;
+
+class DataFlowExpr = Expr;
+
+newtype TDataFlowType =
+  TStringFlow()
+
+class DataFlowType extends TDataFlowType {
+  string toString() { none() }
+}
+
+/** Gets a viable run-time target for the call `call`. */
+DataFlowCallable viableCallable(DataFlowCall call) { none() }
+
+/**
+ * Holds if `t1` and `t2` are compatible, that is, whether data can flow from
+ * a node of type `t1` to a node of type `t2`.
+ */
+pragma[inline]
+predicate compatibleTypes(DataFlowType t1, DataFlowType t2) {
+  none()
+}
+
+/**
+ * This is the local flow predicate that is used as a building block in global
+ * data flow. It is a strict subset of the `localFlowStep` predicate, as it
+ * excludes SSA flow through instance fields.
+ */
+predicate simpleLocalFlowStep(Node nodeFrom, Node nodeTo) {
+  none()
+}
+
+/**
+ * Holds if `pred` can flow to `succ`, by jumping from one callable to
+ * another. Additional steps specified by the configuration are *not*
+ * taken into account.
+ */
+predicate jumpStep(ExprNode pred, ExprNode succ) {
+  none()
+}
+
+/**
+ * Holds if data can flow from `node1` to `node2` via an assignment to
+ * content `c`.
+ */
+predicate storeStep(Node node1, Content c, Node node2) {
+  none()
+}
+
+/**
+ * Holds if data can flow from `node1` to `node2` via a read of content `c`.
+ */
+predicate readStep(Node node1, Content c, Node node2) {
+  none()
+}
+
+/**
+ * Gets a node that can read the value returned from `call` with return kind
+ * `kind`.
+ */
+OutNode getAnOutNode(DataFlowCall call, ReturnKind kind) { call = result.getCall(kind) }
+
+/**
+ * Holds if the call context `ctx` reduces the set of viable run-time
+ * targets of call `call` in `c`.
+ */
+predicate reducedViableImplInCallContext(DataFlowCall call, DataFlowCallable c, DataFlowCall ctx) {
+  none()
+}
+
+/**
+ * Holds if the node `n` is unreachable when the call context is `call`.
+ */
+predicate isUnreachableInCall(Node n, DataFlowCall call) {
+  none()
+}
+
+/**
+ * Holds if flow returning from callable `c` to call `call` might return
+ * further and if this path restricts the set of call sites that can be
+ * returned to.
+ */
+predicate reducedViableImplInReturn(DataFlowCallable c, DataFlowCall call) {
+  none()
+}
+
+/**
+ * Gets a viable run-time target for the call `call` in the context `ctx`.
+ * This is restricted to those call nodes and results for which the return
+ * flow from the result to `call` restricts the possible context `ctx`.
+ */
+DataFlowCallable prunedViableImplInCallContextReverse(DataFlowCall call, DataFlowCall ctx) {
+  none()
+  // result = viableImplInCallContext(call, ctx) and
+  // reducedViableImplInReturn(result, call)
+}
+
+/**
+ * Gets a viable run-time target for the call `call` in the context
+ * `ctx`. This is restricted to those call nodes for which a context
+ * might make a difference.
+ */
+DataFlowCallable prunedViableImplInCallContext(DataFlowCall call, DataFlowCall ctx) {
+  none()
+  // result = viableImplInCallContext(call, ctx) and
+  // reducedViableImplInCallContext(call, _, ctx)
+}
+
+/**
+ * Holds if `n` does not require a `PostUpdateNode` as it either cannot be
+ * modified or its modification cannot be observed, for example if it is a
+ * freshly created object that is not saved in a variable.
+ *
+ * This predicate is only used for consistency checks.
+ */
+predicate isImmutableOrUnobservable(Node n) { none() }
+
+DataFlowType getErasedRepr(DataFlowType t) { result = t }
+
+/** Gets a string representation of a type returned by `getErasedRepr`. */
+string ppReprType(DataFlowType t) { result = t.toString() }
+
+int accessPathLimit() { result = 3 }
+
+/** Holds if `n` should be hidden from path explanations. */
+predicate nodeIsHidden(Node n) { none() }

python/ql/src/semmle/code/python/dataflow/internal/DataFlowPublic.qll

@@ -0,0 +1,95 @@
+import python
+private import DataFlowPrivate
+
+/**
+ * An element, viewed as a node in a data flow graph. Either an expression
+ * (`ExprNode`) or a parameter (`ParameterNode`).
+ */
+class Node extends ControlFlowNode {
+  /** Gets the enclosing callable of this node. */
+  final DataFlowCallable getEnclosingCallable() {
+    none()
+  }
+
+  /**
+   * Gets an upper bound on the type of this node.
+   */
+  DataFlowType getTypeBound() {
+    none()
+  }
+
+  /**
+   * Holds if this element is at the specified location.
+   * The location spans column `startcolumn` of line `startline` to
+   * column `endcolumn` of line `endline` in file `filepath`.
+   * For more information, see
+   * [Locations](https://help.semmle.com/QL/learn-ql/ql/locations.html).
+   */
+  predicate hasLocationInfo(
+    string filepath, int startline, int startcolumn, int endline, int endcolumn
+  ) {
+    getLocation().hasLocationInfo(filepath, startline, startcolumn, endline, endcolumn)
+  }
+
+}
+
+/**
+ * An expression, viewed as a node in a data flow graph.
+ *
+ * Note that because of control-flow splitting, one `Expr` may correspond
+ * to multiple `ExprNode`s, just like it may correspond to multiple
+ * `ControlFlow::Node`s.
+ */
+class ExprNode extends Node {
+}
+
+/** Gets a node corresponding to expression `e`. */
+ExprNode exprNode(DataFlowExpr e) { none() }
+
+/**
+ * The value of a parameter at function entry, viewed as a node in a data
+ * flow graph.
+ */
+class ParameterNode extends Node {
+    /**
+   * Holds if this node is the parameter of callable `c` at the specified
+   * (zero-based) position.
+   */
+  predicate isParameterOf(DataFlowCallable c, int i) { none() }
+}
+
+/**
+ * A guard that validates some expression.
+ *
+ * To use this in a configuration, extend the class and provide a
+ * characteristic predicate precisely specifying the guard, and override
+ * `checks` to specify what is being validated and in which branch.
+ *
+ * It is important that all extending classes in scope are disjoint.
+ */
+class BarrierGuard extends Expr {
+  /** Holds if this guard validates `e` upon evaluating to `v`. */
+  // abstract predicate checks(Expr e, AbstractValue v);
+
+  /** Gets a node guarded by this guard. */
+  final ExprNode getAGuardedNode() {
+    none()
+    // exists(Expr e, AbstractValue v |
+    //   this.checks(e, v) and
+    //   this.controlsNode(result.getControlFlowNode(), e, v)
+    // )
+  }
+}
+
+/**
+ * A reference contained in an object. This is either a field or a property.
+ */
+class Content extends string {
+  Content() { this = "Content" }
+
+  /** Gets the type of the object containing this content. */
+  DataFlowType getContainerType() { none() }
+
+  /** Gets the type of this content. */
+  DataFlowType getType() { none() }
+}

python/ql/src/semmle/code/python/dataflow/internal/TaintTrackingPrivate.qll

@@ -0,0 +1,21 @@
+private import python
+private import TaintTrackingPublic
+private import semmle.code.python.dataflow.DataFlow
+private import semmle.code.python.dataflow.internal.DataFlowPrivate
+
+/**
+ * Holds if `node` should be a barrier in all global taint flow configurations
+ * but not in local taint.
+ */
+predicate defaultTaintBarrier(DataFlow::Node node) { none() }
+
+/**
+ * Holds if the additional step from `src` to `sink` should be included in all
+ * global taint flow configurations.
+ */
+predicate defaultAdditionalTaintStep(DataFlow::Node pred, DataFlow::Node succ) {
+  none()
+  // localAdditionalTaintStep(pred, succ)
+  // or
+  // succ = pred.(DataFlow::NonLocalJumpNode).getAJumpSuccessor(false)
+}

python/ql/src/semmle/code/python/dataflow/internal/TaintTrackingPublic.qll

@@ -0,0 +1,31 @@
+private import python
+private import TaintTrackingPrivate
+private import semmle.code.python.dataflow.DataFlow
+
+// /**
+//  * Holds if taint propagates from `source` to `sink` in zero or more local
+//  * (intra-procedural) steps.
+//  */
+// predicate localTaint(DataFlow::Node source, DataFlow::Node sink) { localTaintStep*(source, sink) }
+
+// /**
+//  * Holds if taint can flow from `e1` to `e2` in zero or more
+//  * local (intra-procedural) steps.
+//  */
+// predicate localExprTaint(Expr e1, Expr e2) {
+//   localTaint(DataFlow::exprNode(e1), DataFlow::exprNode(e2))
+// }
+
+// /** A member (property or field) that is tainted if its containing object is tainted. */
+// abstract class TaintedMember extends AssignableMember { }
+
+// /**
+//  * Holds if taint propagates from `nodeFrom` to `nodeTo` in exactly one local
+//  * (intra-procedural) step.
+//  */
+// predicate localTaintStep(DataFlow::Node nodeFrom, DataFlow::Node nodeTo) {
+//   // Ordinary data flow
+//   DataFlow::localFlowStep(nodeFrom, nodeTo)
+//   or
+//   localAdditionalTaintStep(nodeFrom, nodeTo)
+// }

python/ql/src/semmle/code/python/dataflow/internal/tainttracking1/TaintTrackingImpl.qll

@@ -0,0 +1,115 @@
+/**
+ * Provides an implementation of global (interprocedural) taint tracking.
+ * This file re-exports the local (intraprocedural) taint-tracking analysis
+ * from `TaintTrackingParameter::Public` and adds a global analysis, mainly
+ * exposed through the `Configuration` class. For some languages, this file
+ * exists in several identical copies, allowing queries to use multiple
+ * `Configuration` classes that depend on each other without introducing
+ * mutual recursion among those configurations.
+ */
+
+import TaintTrackingParameter::Public
+private import TaintTrackingParameter::Private
+
+/**
+ * A configuration of interprocedural taint tracking analysis. This defines
+ * sources, sinks, and any other configurable aspect of the analysis. Each
+ * use of the taint tracking library must define its own unique extension of
+ * this abstract class.
+ *
+ * A taint-tracking configuration is a special data flow configuration
+ * (`DataFlow::Configuration`) that allows for flow through nodes that do not
+ * necessarily preserve values but are still relevant from a taint tracking
+ * perspective. (For example, string concatenation, where one of the operands
+ * is tainted.)
+ *
+ * To create a configuration, extend this class with a subclass whose
+ * characteristic predicate is a unique singleton string. For example, write
+ *
+ * ```
+ * class MyAnalysisConfiguration extends TaintTracking::Configuration {
+ *   MyAnalysisConfiguration() { this = "MyAnalysisConfiguration" }
+ *   // Override `isSource` and `isSink`.
+ *   // Optionally override `isSanitizer`.
+ *   // Optionally override `isSanitizerIn`.
+ *   // Optionally override `isSanitizerOut`.
+ *   // Optionally override `isSanitizerGuard`.
+ *   // Optionally override `isAdditionalTaintStep`.
+ * }
+ * ```
+ *
+ * Then, to query whether there is flow between some `source` and `sink`,
+ * write
+ *
+ * ```
+ * exists(MyAnalysisConfiguration cfg | cfg.hasFlow(source, sink))
+ * ```
+ *
+ * Multiple configurations can coexist, but it is unsupported to depend on
+ * another `TaintTracking::Configuration` or a `DataFlow::Configuration` in the
+ * overridden predicates that define sources, sinks, or additional steps.
+ * Instead, the dependency should go to a `TaintTracking2::Configuration` or a
+ * `DataFlow2::Configuration`, `DataFlow3::Configuration`, etc.
+ */
+abstract class Configuration extends DataFlow::Configuration {
+  bindingset[this]
+  Configuration() { any() }
+
+  /**
+   * Holds if `source` is a relevant taint source.
+   *
+   * The smaller this predicate is, the faster `hasFlow()` will converge.
+   */
+  // overridden to provide taint-tracking specific qldoc
+  abstract override predicate isSource(DataFlow::Node source);
+
+  /**
+   * Holds if `sink` is a relevant taint sink.
+   *
+   * The smaller this predicate is, the faster `hasFlow()` will converge.
+   */
+  // overridden to provide taint-tracking specific qldoc
+  abstract override predicate isSink(DataFlow::Node sink);
+
+  /** Holds if the node `node` is a taint sanitizer. */
+  predicate isSanitizer(DataFlow::Node node) { none() }
+
+  final override predicate isBarrier(DataFlow::Node node) {
+    isSanitizer(node) or
+    defaultTaintBarrier(node)
+  }
+
+  /** Holds if data flow into `node` is prohibited. */
+  predicate isSanitizerIn(DataFlow::Node node) { none() }
+
+  final override predicate isBarrierIn(DataFlow::Node node) { isSanitizerIn(node) }
+
+  /** Holds if data flow out of `node` is prohibited. */
+  predicate isSanitizerOut(DataFlow::Node node) { none() }
+
+  final override predicate isBarrierOut(DataFlow::Node node) { isSanitizerOut(node) }
+
+  /** Holds if data flow through nodes guarded by `guard` is prohibited. */
+  predicate isSanitizerGuard(DataFlow::BarrierGuard guard) { none() }
+
+  final override predicate isBarrierGuard(DataFlow::BarrierGuard guard) { isSanitizerGuard(guard) }
+
+  /**
+   * Holds if the additional taint propagation step from `node1` to `node2`
+   * must be taken into account in the analysis.
+   */
+  predicate isAdditionalTaintStep(DataFlow::Node node1, DataFlow::Node node2) { none() }
+
+  final override predicate isAdditionalFlowStep(DataFlow::Node node1, DataFlow::Node node2) {
+    isAdditionalTaintStep(node1, node2) or
+    defaultAdditionalTaintStep(node1, node2)
+  }
+
+  /**
+   * Holds if taint may flow from `source` to `sink` for this configuration.
+   */
+  // overridden to provide taint-tracking specific qldoc
+  override predicate hasFlow(DataFlow::Node source, DataFlow::Node sink) {
+    super.hasFlow(source, sink)
+  }
+}

python/ql/src/semmle/code/python/dataflow/internal/tainttracking1/TaintTrackingParameter.qll

@@ -0,0 +1,6 @@
+import semmle.code.python.dataflow.internal.TaintTrackingPublic as Public
+
+module Private {
+  import semmle.code.python.dataflow.DataFlow::DataFlow as DataFlow
+  import semmle.code.python.dataflow.internal.TaintTrackingPrivate
+}