python: refactor TDataFlowCall

- Branch predicates are made simple. In particular, they do not try to detect library calls.
- All branches based on `CallNode`s are gathered into one.
- That branch has been given a class `NonSpecialCall`, which is the new parent of call classes based on `CallNode`s. (Those classes now have more involved charpreds.)
- A new such class, 'LambdaCall` has been split out from `FunctionCall` to allow the latter to replace its
  general `CallNode` field with a specific `FunctionValue` one.
- `NonSpecialCall` is not an abstract class, but it has some abstract overrides. Therefor, it is not
  considered a resolved call in the test `UnresolvedCalls.qll`.

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowDispatchPointsTo.qll

@@ -421,22 +421,24 @@ class LibraryCallableValue extends DataFlowCallable, TLibraryCallable {
  * TODO: Add `TClassMethodCall` mapping `cls` appropriately.
  */
 newtype TDataFlowCall =
-  TFunctionCall(CallNode call) { call = any(FunctionValue f).getAFunctionCall() } or
-  /** Bound methods need to make room for the explicit self parameter */
-  TMethodCall(CallNode call) { call = any(FunctionValue f).getAMethodCall() } or
-  TClassCall(CallNode call) { call = any(ClassValue c | not c.isAbsent()).getACall() } or
+  /**
+   * Includes function calls, method calls, class calls and library calls.
+   * All these will be associated with a `CallNode`.
+   */
+  TNonSpecialCall(CallNode call) or
+  /**
+   * Includes calls to special methods.
+   * These will be associated with a `SpecialMethodCallNode`.
+   */
   TSpecialCall(SpecialMethodCallNode special) or
-  /** A call to a summarized callable */
-  TLibraryCall(CallNode call) { call.getNode() = any(LibraryCallable lc).getACall() } or
-  /** A synthesized inside a summarized callable */
+  /** A synthesized call inside a summarized callable */
   TSummaryCall(FlowSummaryImpl::Public::SummarizedCallable c, Node receiver) {
     FlowSummaryImpl::Private::summaryCallbackRange(c, receiver)
   }
 
-class TDataFlowSourceCall =
-  TFunctionCall or TMethodCall or TClassCall or TSpecialCall or TLibraryCall;
+class TDataFlowSourceCall = TSpecialCall or TNonSpecialCall;
 
-/** A call. */
+/** A call that is taken into account by the global data flow computation. */
 abstract class DataFlowCall extends TDataFlowCall {
   /** Gets a textual representation of this element. */
   abstract string toString();
@@ -461,6 +463,7 @@ abstract class DataFlowCall extends TDataFlowCall {
   }
 }
 
+/** A call found in the program source (as opposed to a synthesised call). */
 abstract class DataFlowSourceCall extends DataFlowCall, TDataFlowSourceCall {
   final override Location getLocation() { result = this.getNode().getLocation() }
 
@@ -477,60 +480,76 @@ abstract class DataFlowSourceCall extends DataFlowCall, TDataFlowSourceCall {
   abstract ControlFlowNode getNode();
 }
 
+/** A call associated with a `CallNode`. */
+class NonSpecialCall extends DataFlowSourceCall, TNonSpecialCall {
+  CallNode call;
+
+  NonSpecialCall() { this = TNonSpecialCall(call) }
+
+  override string toString() { result = call.toString() }
+
+  abstract override Node getArg(int n);
+
+  override ControlFlowNode getNode() { result = call }
+
+  abstract override DataFlowCallable getCallable();
+
+  override DataFlowCallable getEnclosingCallable() { result.getScope() = call.getNode().getScope() }
+}
+
 /**
- * A call to a function/lambda.
+ * A call to a function.
  * This excludes calls to bound methods, classes, and special methods.
  * Bound method calls and class calls insert an argument for the explicit
  * `self` parameter, and special method calls have special argument passing.
  */
-class FunctionCall extends DataFlowSourceCall, TFunctionCall {
-  CallNode call;
+class FunctionCall extends NonSpecialCall {
   DataFlowCallable callable;
+  FunctionValue f;
 
   FunctionCall() {
-    this = TFunctionCall(call) and
+    call = f.getAFunctionCall() and
     call = callable.getACall()
   }
 
-  override string toString() { result = call.toString() }
+  override Node getArg(int n) { result = getArg(call, TNoShift(), callable.getCallableValue(), n) }
+
+  override DataFlowCallable getCallable() { result = callable }
+}
+
+/** A call to a lambda. */
+class LambdaCall extends NonSpecialCall {
+  DataFlowCallable callable;
+  Function f;
+
+  LambdaCall() {
+    call = callable.getACall() and
+    callable = TLambda(f)
+  }
 
   override Node getArg(int n) { result = getArg(call, TNoShift(), callable.getCallableValue(), n) }
 
-  override ControlFlowNode getNode() { result = call }
-
   override DataFlowCallable getCallable() { result = callable }
-
-  override DataFlowCallable getEnclosingCallable() { result.getScope() = call.getNode().getScope() }
 }
 
 /**
  * Represents a call to a bound method call.
  * The node representing the instance is inserted as argument to the `self` parameter.
  */
-class MethodCall extends DataFlowSourceCall, TMethodCall {
-  CallNode call;
+class MethodCall extends NonSpecialCall {
   FunctionValue bm;
 
-  MethodCall() {
-    this = TMethodCall(call) and
-    call = bm.getACall()
-  }
+  MethodCall() { call = bm.getAMethodCall() }
 
   private CallableValue getCallableValue() { result = bm }
 
-  override string toString() { result = call.toString() }
-
   override Node getArg(int n) {
     n > 0 and result = getArg(call, TShiftOneUp(), this.getCallableValue(), n)
     or
     n = 0 and result = TCfgNode(call.getFunction().(AttrNode).getObject())
   }
 
-  override ControlFlowNode getNode() { result = call }
-
   override DataFlowCallable getCallable() { result = TCallableValue(this.getCallableValue()) }
-
-  override DataFlowCallable getEnclosingCallable() { result.getScope() = call.getScope() }
 }
 
 /**
@@ -539,30 +558,23 @@ class MethodCall extends DataFlowSourceCall, TMethodCall {
  * That makes the call node be the post-update node holding the value of the object
  * after the constructor has run.
  */
-class ClassCall extends DataFlowSourceCall, TClassCall {
-  CallNode call;
+class ClassCall extends NonSpecialCall {
   ClassValue c;
 
   ClassCall() {
-    this = TClassCall(call) and
+    not c.isAbsent() and
     call = c.getACall()
   }
 
   private CallableValue getCallableValue() { c.getScope().getInitMethod() = result.getScope() }
 
-  override string toString() { result = call.toString() }
-
   override Node getArg(int n) {
     n > 0 and result = getArg(call, TShiftOneUp(), this.getCallableValue(), n)
     or
     n = 0 and result = TSyntheticPreUpdateNode(TCfgNode(call))
   }
 
-  override ControlFlowNode getNode() { result = call }
-
   override DataFlowCallable getCallable() { result = TCallableValue(this.getCallableValue()) }
-
-  override DataFlowCallable getEnclosingCallable() { result.getScope() = call.getScope() }
 }
 
 /** A call to a special method. */
@@ -586,22 +598,16 @@ class SpecialCall extends DataFlowSourceCall, TSpecialCall {
   }
 }
 
-class LibraryCall extends DataFlowSourceCall, TLibraryCall {
-  CallNode call;
+/** A call to a summarized callable. */
+class LibraryCall extends NonSpecialCall {
   LibraryCallable callable;
 
-  LibraryCall() { this = TLibraryCall(call) and call.getNode() = callable.getACall() }
-
-  override string toString() { result = call.toString() }
+  LibraryCall() { call.getNode() = callable.getACall() }
 
   // TODO: Implement Python calling convention?
   override Node getArg(int n) { result = TCfgNode(call.getArg(n)) }
 
-  override ControlFlowNode getNode() { result = call }
-
   override DataFlowCallable getCallable() { result.asLibraryCallable() = callable }
-
-  override DataFlowCallable getEnclosingCallable() { result.getScope() = call.getNode().getScope() }
 }
 
 /**

python/ql/test/experimental/dataflow/TestUtil/UnresolvedCalls.qll

@@ -12,7 +12,13 @@ class UnresolvedCallExpectations extends InlineExpectationsTest {
   override predicate hasActualResult(Location location, string element, string tag, string value) {
     exists(location.getFile().getRelativePath()) and
     exists(CallNode call |
-      not exists(DataFlowPrivate::DataFlowSourceCall dfc | dfc.getNode() = call) and
+      not exists(DataFlowPrivate::DataFlowSourceCall dfc | dfc.getNode() = call |
+        // For every `CallNode`, there is a `DataFlowSourceCall` in the form of a `NonSpecialCall`.
+        // It does not really count, as it has some abstract overrides. For instance, it does not
+        // define `getCallable`, so checking for the existence of this guarantees that we are in a
+        // properly resolved call.
+        exists(dfc.getCallable())
+      ) and
       not call = API::builtin(_).getACall().asCfgNode() and
       location = call.getLocation() and
       tag = "unresolved_call" and

python/ql/test/experimental/dataflow/calls/test.py

@@ -31,7 +31,7 @@ try:
     # `mypkg.foo` is a `missing module variable`, but `mypkg.subpkg.bar` is compeltely
     # ignored.
     import mypkg
-    mypkg.foo(42)
-    mypkg.subpkg.bar(43)
+    mypkg.foo(42) # $ call=mypkg.foo(..) qlclass=NonSpecialCall
+    mypkg.subpkg.bar(43) # $ call=mypkg.subpkg.bar(..) qlclass=NonSpecialCall
 except:
     pass

python/ql/test/experimental/dataflow/coverage/classesCallGraph.expected

@@ -1,3 +1,4 @@
+| classes.py:45:16:45:35 | ControlFlowNode for Attribute() | classes.py:45:16:45:35 | ControlFlowNode for Attribute() |
 | classes.py:60:17:60:27 | [pre objCreate] ControlFlowNode for With_init() | classes.py:54:18:54:21 | ControlFlowNode for self |
 | classes.py:242:9:242:24 | ControlFlowNode for set() | classes.py:242:9:242:24 | ControlFlowNode for set() |
 | classes.py:247:9:247:30 | ControlFlowNode for frozenset() | classes.py:247:9:247:30 | ControlFlowNode for frozenset() |