Merge pull request #11376 from RasmusWL/call-graph-code

Python: New type-tracking based call-graph

python/ql/src/Security/CWE-020-ExternalAPIs/ExternalAPIs.qll

@@ -1,48 +1,36 @@
 /**
  * Definitions for reasoning about untrusted data used in APIs defined outside the
- * database.
+ * user-written code.
  */
 
-import python
+private import python
 import semmle.python.dataflow.new.DataFlow
-import semmle.python.dataflow.new.TaintTracking
-import semmle.python.Concepts
-import semmle.python.dataflow.new.RemoteFlowSources
+private import semmle.python.dataflow.new.TaintTracking
+private import semmle.python.dataflow.new.RemoteFlowSources
+private import semmle.python.ApiGraphs
 private import semmle.python.dataflow.new.internal.DataFlowPrivate as DataFlowPrivate
 private import semmle.python.dataflow.new.internal.TaintTrackingPrivate as TaintTrackingPrivate
-private import semmle.python.types.Builtins
-private import semmle.python.objects.ObjectInternal
 
-// IMPLEMENTATION NOTES:
-//
-// This query uses *both* the new data-flow library, and points-to. Why? To get this
-// finished quickly, so it can provide value for our field team and ourselves.
-//
-// In the long run, it should not need to use points-to for anything. Possibly this can
-// even be helpful in figuring out what we need from TypeTrackers and the new data-flow
-// library to be fully operational.
-//
-// At least it will allow us to provide a baseline comparison against a solution that
-// doesn't use points-to at all
-//
-// There is a few dirty things we do here:
-// 1. DataFlowPrivate: since `DataFlowCall` and `DataFlowCallable` are not exposed
-//    publicly, but we really want access to them.
-// 2. points-to: we kinda need to do this since this is what powers `DataFlowCall` and
-//    `DataFlowCallable`
-// 3. ObjectInternal: to provide better names for built-in functions and methods. If we
-//    really wanted to polish our points-to implementation, we could move this
-//    functionality into `BuiltinFunctionValue` and `BuiltinMethodValue`, but will
-//    probably require some more work: for this query, it's totally ok to use
-//    `builtins.open` for the code `open(f)`, but well, it requires a bit of thinking to
-//    figure out if that is desirable in general. I simply skipped a corner here!
-// 4. TaintTrackingPrivate: Nothing else gives us access to `defaultAdditionalTaintStep` :(
 /**
- * A callable that is considered a "safe" external API from a security perspective.
+ * An external API that is considered "safe" from a security perspective.
  */
 class SafeExternalApi extends Unit {
-  /** Gets a callable that is considered a "safe" external API from a security perspective. */
-  abstract DataFlowPrivate::DataFlowCallable getSafeCallable();
+  /**
+   * Gets a call that is considered "safe" from a security perspective. You can use API
+   * graphs to find calls to functions you know are safe.
+   *
+   * Which works even when the external library isn't extracted.
+   */
+  abstract DataFlow::CallCfgNode getSafeCall();
+
+  /**
+   * Gets a callable that is considered a "safe" external API from a security
+   * perspective.
+   *
+   * You probably want to define this as `none()` and use `getSafeCall` instead, since
+   * that can handle the external library not being extracted.
+   */
+  DataFlowPrivate::DataFlowCallable getSafeCallable() { none() }
 }
 
 /** DEPRECATED: Alias for SafeExternalApi */
@@ -50,42 +38,127 @@ deprecated class SafeExternalAPI = SafeExternalApi;
 
 /** The default set of "safe" external APIs. */
 private class DefaultSafeExternalApi extends SafeExternalApi {
-  override DataFlowPrivate::DataFlowCallable getSafeCallable() {
-    exists(CallableValue cv | cv = result.getCallableValue() |
-      cv = Value::named(["len", "isinstance", "getattr", "hasattr"])
-      or
-      exists(ClassValue cls, string attr |
-        cls = Value::named("dict") and attr in ["__getitem__", "__setitem__"]
-      |
-        cls.lookup(attr) = cv
-      )
+  override DataFlow::CallCfgNode getSafeCall() {
+    result =
+      API::builtin([
+          "len", "enumerate", "isinstance", "getattr", "hasattr", "bool", "float", "int", "repr",
+          "str", "type"
+        ]).getACall()
+  }
+}
+
+/**
+ * Gets a human readable representation of `node`.
+ *
+ * Note that this is only defined for API nodes that are allowed as external APIs,
+ * so `None.json.dumps` will for example not be allowed.
+ */
+string apiNodeToStringRepr(API::Node node) {
+  node = API::builtin(result)
+  or
+  node = API::moduleImport(result)
+  or
+  exists(API::Node base, string basename |
+    base.getDepth() < node.getDepth() and
+    basename = apiNodeToStringRepr(base) and
+    not base = API::builtin(["None", "True", "False"])
+  |
+    exists(string m | node = base.getMember(m) | result = basename + "." + m)
+    or
+    node = base.getReturn() and
+    result = basename + "()" and
+    not base.getACall() = any(SafeExternalApi safe).getSafeCall()
+    or
+    node = base.getAwaited() and
+    result = basename
+  )
+}
+
+predicate resolvedCall(CallNode call) {
+  DataFlowPrivate::resolveCall(call, _, _) or
+  DataFlowPrivate::resolveClassCall(call, _)
+}
+
+newtype TInterestingExternalApiCall =
+  TUnresolvedCall(DataFlow::CallCfgNode call) {
+    exists(call.getLocation().getFile().getRelativePath()) and
+    not resolvedCall(call.getNode()) and
+    not call = any(SafeExternalApi safe).getSafeCall()
+  } or
+  TResolvedCall(DataFlowPrivate::DataFlowCall call) {
+    exists(call.getLocation().getFile().getRelativePath()) and
+    exists(call.getCallable()) and
+    not call.getCallable() = any(SafeExternalApi safe).getSafeCallable() and
+    // ignore calls inside codebase, and ignore calls that are marked as  safe. This is
+    // only needed as long as we extract dependencies. When we stop doing that, all
+    // targets of resolved calls will be from user-written code.
+    not exists(call.getCallable().getLocation().getFile().getRelativePath()) and
+    not exists(DataFlow::CallCfgNode callCfgNode | callCfgNode.getNode() = call.getNode() |
+      any(SafeExternalApi safe).getSafeCall() = callCfgNode
+    )
+  }
+
+abstract class InterestingExternalApiCall extends TInterestingExternalApiCall {
+  /** Gets the argument at position `apos`, if any */
+  abstract DataFlow::Node getArgument(DataFlowPrivate::ArgumentPosition apos);
+
+  /** Gets a textual representation of this element. */
+  abstract string toString();
+
+  /**
+   * Gets a human-readable name for the external API.
+   */
+  abstract string getApiName();
+}
+
+class UnresolvedCall extends InterestingExternalApiCall, TUnresolvedCall {
+  DataFlow::CallCfgNode call;
+
+  UnresolvedCall() { this = TUnresolvedCall(call) }
+
+  override DataFlow::Node getArgument(DataFlowPrivate::ArgumentPosition apos) {
+    exists(int i | apos.isPositional(i) | result = call.getArg(i))
+    or
+    exists(string name | apos.isKeyword(name) | result = call.getArgByName(name))
+  }
+
+  override string toString() {
+    result = "ExternalAPI:UnresolvedCall: " + call.getNode().getNode().toString()
+  }
+
+  override string getApiName() {
+    exists(API::Node apiNode |
+      result = apiNodeToStringRepr(apiNode) and
+      apiNode.getACall() = call
+    )
+  }
+}
+
+class ResolvedCall extends InterestingExternalApiCall, TResolvedCall {
+  DataFlowPrivate::DataFlowCall dfCall;
+
+  ResolvedCall() { this = TResolvedCall(dfCall) }
+
+  override DataFlow::Node getArgument(DataFlowPrivate::ArgumentPosition apos) {
+    result = dfCall.getArgument(apos)
+  }
+
+  override string toString() {
+    result = "ExternalAPI:ResolvedCall: " + dfCall.getNode().getNode().toString()
+  }
+
+  override string getApiName() {
+    exists(DataFlow::CallCfgNode call, API::Node apiNode | dfCall.getNode() = call.getNode() |
+      result = apiNodeToStringRepr(apiNode) and
+      apiNode.getACall() = call
     )
   }
 }
 
 /** A node representing data being passed to an external API through a call. */
 class ExternalApiDataNode extends DataFlow::Node {
-  DataFlowPrivate::DataFlowCallable callable;
-  int i;
-
   ExternalApiDataNode() {
-    exists(DataFlowPrivate::DataFlowCall call |
-      exists(call.getLocation().getFile().getRelativePath())
-    |
-      callable = call.getCallable() and
-      // TODO: this ignores some complexity of keyword arguments (especially keyword-only args)
-      this = call.getArg(i)
-    ) and
-    not any(SafeExternalApi safe).getSafeCallable() = callable and
-    exists(Value cv | cv = callable.getCallableValue() |
-      cv.isAbsent()
-      or
-      cv.isBuiltin()
-      or
-      cv.(CallableValue).getScope().getLocation().getFile().inStdlib()
-      or
-      not exists(cv.(CallableValue).getScope().getLocation().getFile().getRelativePath())
-    ) and
+    exists(InterestingExternalApiCall call | this = call.getArgument(_)) and
     // Not already modeled as a taint step
     not TaintTrackingPrivate::defaultAdditionalTaintStep(this, _) and
     // for `list.append(x)`, we have a additional taint step from x -> [post] list.
@@ -95,12 +168,6 @@ class ExternalApiDataNode extends DataFlow::Node {
       TaintTrackingPrivate::defaultAdditionalTaintStep(_, post)
     )
   }
-
-  /** Gets the index for the parameter that will receive this untrusted data */
-  int getIndex() { result = i }
-
-  /** Gets the callable to which this argument is passed. */
-  DataFlowPrivate::DataFlowCallable getCallable() { result = callable }
 }
 
 /** DEPRECATED: Alias for ExternalApiDataNode */
@@ -133,19 +200,26 @@ deprecated class UntrustedExternalAPIDataNode = UntrustedExternalApiDataNode;
 
 /** An external API which is used with untrusted data. */
 private newtype TExternalApi =
-  /** An untrusted API method `m` where untrusted data is passed at `index`. */
-  TExternalApiParameter(DataFlowPrivate::DataFlowCallable callable, int index) {
-    exists(UntrustedExternalApiDataNode n |
-      callable = n.getCallable() and
-      index = n.getIndex()
+  MkExternalApi(string repr, DataFlowPrivate::ArgumentPosition apos) {
+    exists(UntrustedExternalApiDataNode ex, InterestingExternalApiCall call |
+      ex = call.getArgument(apos) and
+      repr = call.getApiName()
     )
   }
 
-/** An external API which is used with untrusted data. */
-class ExternalApiUsedWithUntrustedData extends TExternalApi {
+/** A argument of an external API which is used with untrusted data. */
+class ExternalApiUsedWithUntrustedData extends MkExternalApi {
+  string repr;
+  DataFlowPrivate::ArgumentPosition apos;
+
+  ExternalApiUsedWithUntrustedData() { this = MkExternalApi(repr, apos) }
+
   /** Gets a possibly untrusted use of this external API. */
   UntrustedExternalApiDataNode getUntrustedDataNode() {
-    this = TExternalApiParameter(result.getCallable(), result.getIndex())
+    exists(InterestingExternalApiCall call |
+      result = call.getArgument(apos) and
+      call.getApiName() = repr
+    )
   }
 
   /** Gets the number of untrusted sources used with this external API. */
@@ -154,63 +228,8 @@ class ExternalApiUsedWithUntrustedData extends TExternalApi {
   }
 
   /** Gets a textual representation of this element. */
-  string toString() {
-    exists(
-      DataFlowPrivate::DataFlowCallable callable, int index, string callableString,
-      string indexString
-    |
-      this = TExternalApiParameter(callable, index) and
-      indexString = "param " + index and
-      exists(CallableValue cv | cv = callable.getCallableValue() |
-        callableString =
-          cv.getScope().getEnclosingModule().getName() + "." + cv.getScope().getQualifiedName()
-        or
-        not exists(cv.getScope()) and
-        (
-          cv instanceof BuiltinFunctionValue and
-          callableString = pretty_builtin_function_value(cv)
-          or
-          cv instanceof BuiltinMethodValue and
-          callableString = pretty_builtin_method_value(cv)
-          or
-          not cv instanceof BuiltinFunctionValue and
-          not cv instanceof BuiltinMethodValue and
-          callableString = cv.toString()
-        )
-      ) and
-      result = callableString + " [" + indexString + "]"
-    )
-  }
+  string toString() { result = repr + " [" + apos + "]" }
 }
 
 /** DEPRECATED: Alias for ExternalApiUsedWithUntrustedData */
 deprecated class ExternalAPIUsedWithUntrustedData = ExternalApiUsedWithUntrustedData;
-
-/** Gets the fully qualified name for the `BuiltinFunctionValue` bfv. */
-private string pretty_builtin_function_value(BuiltinFunctionValue bfv) {
-  exists(Builtin b | b = bfv.(BuiltinFunctionObjectInternal).getBuiltin() |
-    result = prefix_with_module_if_found(b)
-  )
-}
-
-/** Gets the fully qualified name for the `BuiltinMethodValue` bmv. */
-private string pretty_builtin_method_value(BuiltinMethodValue bmv) {
-  exists(Builtin b | b = bmv.(BuiltinMethodObjectInternal).getBuiltin() |
-    exists(Builtin cls | cls.isClass() and cls.getMember(b.getName()) = b |
-      result = prefix_with_module_if_found(cls) + "." + b.getName()
-    )
-    or
-    not exists(Builtin cls | cls.isClass() and cls.getMember(b.getName()) = b) and
-    result = b.getName()
-  )
-}
-
-/** Helper predicate that tries to adds module qualifier to `b`. Will succeed even if module not found. */
-private string prefix_with_module_if_found(Builtin b) {
-  exists(Builtin mod | mod.isModule() and mod.getMember(b.getName()) = b |
-    result = mod.getName() + "." + b.getName()
-  )
-  or
-  not exists(Builtin mod | mod.isModule() and mod.getMember(b.getName()) = b) and
-  result = b.getName()
-}

python/ql/src/Security/CWE-020-ExternalAPIs/ExternalAPIsUsedWithUntrustedData.qhelp

@@ -11,11 +11,9 @@ relevant for security analysis of this application.</p>
 
 <p>An external API is defined as a call to a method that is not defined in the source
 code, and is not modeled as a taint step in the default taint library. External APIs may
-be from the Python standard library or dependencies. The query will report the fully qualified name,
-along with <code>[param x]</code>, where <code>x</code> indicates the position of
-the parameter receiving the untrusted data. Note that for methods and
-<code>classmethod</code>s, parameter 0 represents the class instance or class itself
-respectively.</p>
+be from the Python standard library or dependencies. The query will report the fully
+qualified name, along with <code>[position index]</code> or <code>[keyword name]</code>,
+to indicate the argument passing the untrusted data.</p>
 
 <p>Note that an excepted sink might not be included in the results, if it also defines a
 taint step. This is the case for <code>pickle.loads</code> which is a sink for the
@@ -24,8 +22,6 @@ Unsafe Deserialization query, but is also a taint step for other queries.</p>
 <p>Note: Compared to the Java version of this query, we currently do not give special
 care to methods that are overridden in the source code.</p>
 
-<p>Note: Currently this query will only report results for external packages that are extracted.</p>
-
 </overview>
 <recommendation>
 

python/ql/src/Security/CWE-020-ExternalAPIs/UntrustedDataToExternalAPI.qhelp

@@ -11,11 +11,9 @@ be modeled as either taint steps, or sinks for specific problems.</p>
 
 <p>An external API is defined as a call to a method that is not defined in the source
 code, and is not modeled as a taint step in the default taint library. External APIs may
-be from the Python standard library or dependencies. The query will report the fully qualified name,
-along with <code>[param x]</code>, where <code>x</code> indicates the position of
-the parameter receiving the untrusted data. Note that for methods and
-<code>classmethod</code>s, parameter 0 represents the class instance or class itself
-respectively.</p>
+be from the Python standard library or dependencies. The query will report the fully
+qualified name, along with <code>[position index]</code> or <code>[keyword name]</code>,
+to indicate the argument passing the untrusted data.</p>
 
 <p>Note that an excepted sink might not be included in the results, if it also defines a
 taint step. This is the case for <code>pickle.loads</code> which is a sink for the
@@ -24,8 +22,6 @@ Unsafe Deserialization query, but is also a taint step for other queries.</p>
 <p>Note: Compared to the Java version of this query, we currently do not give special
 care to methods that are overridden in the source code.</p>
 
-<p>Note: Currently this query will only report results for external packages that are extracted.</p>
-
 </overview>
 <recommendation>
 

python/ql/src/experimental/semmle/python/security/InsecureRandomnessCustomizations.qll

@@ -59,12 +59,11 @@ module InsecureRandomness {
    */
   class RandomFnSink extends Sink {
     RandomFnSink() {
-      exists(DataFlowCallable randomFn |
-        randomFn
-            .getName()
+      exists(Function func |
+        func.getName()
             .regexpMatch("(?i).*(gen(erate)?|make|mk|create).*(nonce|salt|pepper|Password).*")
       |
-        this.getEnclosingCallable() = randomFn
+        this.asExpr().getScope() = func
       )
     }
   }

python/ql/src/meta/analysis-quality/CallGraph.ql

@@ -1,9 +1,9 @@
 /**
  * @name Call graph
- * @description An edge in the points-to call graph.
+ * @description An edge in the call graph.
  * @kind problem
  * @problem.severity recommendation
- * @id py/meta/points-to-call-graph
+ * @id py/meta/call-graph
  * @tags meta
  * @precision very-low
  */
@@ -12,9 +12,9 @@ import python
 import semmle.python.dataflow.new.internal.DataFlowPrivate
 import meta.MetaMetrics
 
-from DataFlowCall c, DataFlowCallableValue f
+from DataFlowCall call, DataFlowCallable target
 where
-  c.getCallable() = f and
-  not c.getLocation().getFile() instanceof IgnoredFile and
-  not f.getScope().getLocation().getFile() instanceof IgnoredFile
-select c, "Call to $@", f.getScope(), f.toString()
+  target = viableCallable(call) and
+  not call.getLocation().getFile() instanceof IgnoredFile and
+  not target.getScope().getLocation().getFile() instanceof IgnoredFile
+select call, "Call to $@", target.getScope(), target.toString()

python/ql/src/meta/analysis-quality/CallGraphQuality.qll

@@ -1,16 +1,55 @@
 /**
  * Provides predicates for measuring the quality of the call graph, that is,
- * the number of calls that could be resolved to a callee.
+ * the number of calls that could be resolved to a target.
  */
 
 import python
 import meta.MetaMetrics
 
+newtype TTarget =
+  TFunction(Function func) or
+  TClass(Class cls)
+
+class Target extends TTarget {
+  /** Gets a textual representation of this element. */
+  abstract string toString();
+
+  /** Gets the location of this dataflow call. */
+  abstract Location getLocation();
+
+  /** Whether this target is relevant. */
+  predicate isRelevant() { exists(this.getLocation().getFile().getRelativePath()) }
+}
+
+class TargetFunction extends Target, TFunction {
+  Function func;
+
+  TargetFunction() { this = TFunction(func) }
+
+  override string toString() { result = func.toString() }
+
+  override Location getLocation() { result = func.getLocation() }
+
+  Function getFunction() { result = func }
+}
+
+class TargetClass extends Target, TClass {
+  Class cls;
+
+  TargetClass() { this = TClass(cls) }
+
+  override string toString() { result = cls.toString() }
+
+  override Location getLocation() { result = cls.getLocation() }
+
+  Class getClass() { result = cls }
+}
+
 /**
  * A call that is (possibly) relevant for analysis quality.
  * See `IgnoredFile` for details on what is excluded.
  */
-class RelevantCall extends Call {
+class RelevantCall extends CallNode {
   RelevantCall() { not this.getLocation().getFile() instanceof IgnoredFile }
 }
 
@@ -18,12 +57,16 @@ class RelevantCall extends Call {
 module PointsToBasedCallGraph {
   /** A call that can be resolved by points-to. */
   class ResolvableCall extends RelevantCall {
-    Value callee;
+    Value targetValue;
 
-    ResolvableCall() { callee.getACall() = this.getAFlowNode() }
+    ResolvableCall() { targetValue.getACall() = this }
 
-    /** Gets a resolved callee of this call. */
-    Value getCallee() { result = callee }
+    /** Gets a resolved target of this call. */
+    Target getTarget() {
+      result.(TargetFunction).getFunction() = targetValue.(CallableValue).getScope()
+      or
+      result.(TargetClass).getClass() = targetValue.(ClassValue).getScope()
+    }
   }
 
   /** A call that cannot be resolved by points-to. */
@@ -32,34 +75,79 @@ module PointsToBasedCallGraph {
   }
 
   /**
-   * A call that can be resolved by points-to, where the resolved callee is relevant.
-   * Relevant callees include:
-   * - builtins
-   * - standard library
+   * A call that can be resolved by points-to, where the resolved target is relevant.
+   * Relevant targets include:
    * - source code of the project
    */
-  class ResolvableCallRelevantCallee extends ResolvableCall {
-    ResolvableCallRelevantCallee() {
-      callee.isBuiltin()
-      or
-      exists(File file |
-        file = callee.(CallableValue).getScope().getLocation().getFile()
-        or
-        file = callee.(ClassValue).getScope().getLocation().getFile()
-      |
-        file.inStdlib()
-        or
-        // part of the source code of the project
-        exists(file.getRelativePath())
+  class ResolvableCallRelevantTarget extends ResolvableCall {
+    ResolvableCallRelevantTarget() {
+      exists(Target target | target = this.getTarget() |
+        exists(target.getLocation().getFile().getRelativePath())
       )
     }
   }
 
   /**
-   * A call that can be resolved by points-to, where the resolved callee is not considered relevant.
-   * See `ResolvableCallRelevantCallee` for the definition of relevance.
+   * A call that can be resolved by points-to, where the resolved target is not considered relevant.
+   * See `ResolvableCallRelevantTarget` for the definition of relevance.
    */
-  class ResolvableCallIrrelevantCallee extends ResolvableCall {
-    ResolvableCallIrrelevantCallee() { not this instanceof ResolvableCallRelevantCallee }
+  class ResolvableCallIrrelevantTarget extends ResolvableCall {
+    ResolvableCallIrrelevantTarget() { not this instanceof ResolvableCallRelevantTarget }
+  }
+}
+
+/** Provides classes for call-graph resolution by using type-tracking. */
+module TypeTrackingBasedCallGraph {
+  private import semmle.python.dataflow.new.internal.DataFlowDispatch as TT
+
+  /** A call that can be resolved by type-tracking. */
+  class ResolvableCall extends RelevantCall {
+    ResolvableCall() {
+      exists(TT::TNormalCall(this, _, _))
+      or
+      TT::resolveClassCall(this, _)
+    }
+
+    /** Gets a resolved target of this call. */
+    Target getTarget() {
+      exists(TT::DataFlowCall call, TT::CallType ct, Function targetFunc |
+        call = TT::TNormalCall(this, targetFunc, ct) and
+        not ct instanceof TT::CallTypeClass and
+        targetFunc = result.(TargetFunction).getFunction()
+      )
+      or
+      // a TT::TNormalCall only exists when the call can be resolved to a function.
+      // Since points-to just says the call goes directly to the class itself, and
+      // type-tracking based wants to resolve this to the constructor, which might not
+      // exist. So to do a proper comparison, we don't require the call to be resolve to
+      // a specific function.
+      TT::resolveClassCall(this, result.(TargetClass).getClass())
+    }
+  }
+
+  /** A call that cannot be resolved by type-tracking. */
+  class UnresolvableCall extends RelevantCall {
+    UnresolvableCall() { not this instanceof ResolvableCall }
+  }
+
+  /**
+   * A call that can be resolved by type-tracking, where the resolved callee is relevant.
+   * Relevant targets include:
+   * - source code of the project
+   */
+  class ResolvableCallRelevantTarget extends ResolvableCall {
+    ResolvableCallRelevantTarget() {
+      exists(Target target | target = this.getTarget() |
+        exists(target.getLocation().getFile().getRelativePath())
+      )
+    }
+  }
+
+  /**
+   * A call that can be resolved by type-tracking, where the resolved target is not considered relevant.
+   * See `ResolvableCallRelevantTarget` for the definition of relevance.
+   */
+  class ResolvableCallIrrelevantTarget extends ResolvableCall {
+    ResolvableCallIrrelevantTarget() { not this instanceof ResolvableCallRelevantTarget }
   }
 }

python/ql/src/meta/analysis-quality/PointsToResolvableCallsRelevantTarget.ql

@@ -11,4 +11,4 @@
 import python
 import CallGraphQuality
 
-select projectRoot(), count(PointsToBasedCallGraph::ResolvableCallRelevantCallee call)
+select projectRoot(), count(PointsToBasedCallGraph::ResolvableCallRelevantTarget call)

python/ql/src/meta/analysis-quality/TTCallGraph.ql

@@ -0,0 +1,17 @@
+/**
+ * @name New call graph edge from using type-tracking instead of points-to
+ * @kind problem
+ * @problem.severity recommendation
+ * @id py/meta/type-tracking-call-graph
+ * @tags meta
+ * @precision very-low
+ */
+
+import python
+import CallGraphQuality
+
+from CallNode call, Target target
+where
+  target.isRelevant() and
+  call.(TypeTrackingBasedCallGraph::ResolvableCall).getTarget() = target
+select call, "$@ to $@", call, "Call", target, target.toString()

python/ql/src/meta/analysis-quality/TTCallGraphMissing.ql

@@ -0,0 +1,18 @@
+/**
+ * @name Missing call graph edge from using type-tracking instead of points-to
+ * @kind problem
+ * @problem.severity recommendation
+ * @id py/meta/call-graph-missing
+ * @tags meta
+ * @precision very-low
+ */
+
+import python
+import CallGraphQuality
+
+from CallNode call, Target target
+where
+  target.isRelevant() and
+  call.(PointsToBasedCallGraph::ResolvableCall).getTarget() = target and
+  not call.(TypeTrackingBasedCallGraph::ResolvableCall).getTarget() = target
+select call, "MISSING: $@ to $@", call, "Call", target, target.toString()

python/ql/src/meta/analysis-quality/TTCallGraphNew.ql

@@ -0,0 +1,18 @@
+/**
+ * @name New call graph edge from using type-tracking instead of points-to
+ * @kind problem
+ * @problem.severity recommendation
+ * @id py/meta/call-graph-new
+ * @tags meta
+ * @precision very-low
+ */
+
+import python
+import CallGraphQuality
+
+from CallNode call, Target target
+where
+  target.isRelevant() and
+  not call.(PointsToBasedCallGraph::ResolvableCall).getTarget() = target and
+  call.(TypeTrackingBasedCallGraph::ResolvableCall).getTarget() = target
+select call, "NEW: $@ to $@", call, "Call", target, target.toString()

python/ql/src/meta/analysis-quality/TTCallGraphNewAmbiguous.ql

@@ -0,0 +1,19 @@
+/**
+ * @name New call graph edge from using type-tracking instead of points-to, that is ambiguous
+ * @kind problem
+ * @problem.severity recommendation
+ * @id py/meta/call-graph-new-ambiguous
+ * @tags meta
+ * @precision very-low
+ */
+
+import python
+import CallGraphQuality
+
+from CallNode call, Target target
+where
+  target.isRelevant() and
+  not call.(PointsToBasedCallGraph::ResolvableCall).getTarget() = target and
+  call.(TypeTrackingBasedCallGraph::ResolvableCall).getTarget() = target and
+  1 < count(call.(TypeTrackingBasedCallGraph::ResolvableCall).getTarget())
+select call, "NEW: $@ to $@", call, "Call", target, target.toString()

python/ql/src/meta/analysis-quality/TTCallGraphOverview.ql

@@ -0,0 +1,35 @@
+/**
+ * @name Call graph edge overview from using type-tracking instead of points-to
+ * @id py/meta/call-graph-overview
+ * @precision very-low
+ */
+
+import python
+import CallGraphQuality
+
+from string tag, int c
+where
+  tag = "SHARED" and
+  c =
+    count(CallNode call, Target target |
+      target.isRelevant() and
+      call.(PointsToBasedCallGraph::ResolvableCall).getTarget() = target and
+      call.(TypeTrackingBasedCallGraph::ResolvableCall).getTarget() = target
+    )
+  or
+  tag = "NEW" and
+  c =
+    count(CallNode call, Target target |
+      target.isRelevant() and
+      not call.(PointsToBasedCallGraph::ResolvableCall).getTarget() = target and
+      call.(TypeTrackingBasedCallGraph::ResolvableCall).getTarget() = target
+    )
+  or
+  tag = "MISSING" and
+  c =
+    count(CallNode call, Target target |
+      target.isRelevant() and
+      call.(PointsToBasedCallGraph::ResolvableCall).getTarget() = target and
+      not call.(TypeTrackingBasedCallGraph::ResolvableCall).getTarget() = target
+    )
+select tag, c

python/ql/src/meta/analysis-quality/TTCallGraphShared.ql

@@ -0,0 +1,18 @@
+/**
+ * @name Shared call graph edge from using type-tracking instead of points-to
+ * @kind problem
+ * @problem.severity recommendation
+ * @id py/meta/call-graph-shared
+ * @tags meta
+ * @precision very-low
+ */
+
+import python
+import CallGraphQuality
+
+from CallNode call, Target target
+where
+  target.isRelevant() and
+  call.(PointsToBasedCallGraph::ResolvableCall).getTarget() = target and
+  call.(TypeTrackingBasedCallGraph::ResolvableCall).getTarget() = target
+select call, "SHARED: $@ to $@", call, "Call", target, target.toString()