Merge pull request #1669 from markshannon/python-better-handling-unknown-decorators

Python: Treat the result of calling a missing module member as 'unknown'.

change-notes/1.22/analysis-python.md

@@ -3,12 +3,13 @@
 
 ## General improvements
 
+### Points-to
+Tracking of "unknown" values from modules that are absent from the database has been improved. Particularly when an "unknown" value is used as a decorator, the decorated function is tracked.
 
 
 ### Impact on existing queries.
 
 
-
 ## New queries
 
 | **Query** | **Tags** | **Purpose** |

python/ql/src/semmle/python/objects/Modules.qll

@@ -367,12 +367,10 @@ class AbsentModuleAttributeObjectInternal extends ObjectInternal, TAbsentModuleA
     }
 
     override predicate callResult(ObjectInternal obj, CfgOrigin origin) {
-        // Don't know, assume not callable.
-        none()
+        obj = ObjectInternal::unknown() and origin = CfgOrigin::unknown()
     }
 
     override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) {
-        // Don't know, assume not callable.
         none()
     }
 

python/ql/src/semmle/python/objects/ObjectInternal.qll

@@ -66,11 +66,17 @@ class ObjectInternal extends TObject {
 
     /** Holds if `obj` is the result of calling `this` and `origin` is
      * the origin of `obj`.
+     *
+     * This is the context-insensitive version.
+     * Generally, if this holds for any object `obj` then `callResult/3` should never hold for that object.
      */
     abstract predicate callResult(ObjectInternal obj, CfgOrigin origin);
 
     /** Holds if `obj` is the result of calling `this` and `origin` is
      * the origin of `obj` with callee context `callee`.
+     *
+     * This is the context-sensitive version.
+     * Generally, if this holds for any object `obj` then `callResult/2` should never hold for that object.
      */
     abstract predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin);
 

python/ql/test/library-tests/PointsTo/absent/Absent.expected

@@ -5,3 +5,23 @@
 | absent.py:8:1:8:4 | ControlFlowNode for open | file://:0:0:0:0 | Missing module attribute xxxx.open |
 | absent.py:12:8:12:13 | ControlFlowNode for ImportExpr | module.py:0:0:0:0 | Module module |
 | absent.py:14:1:14:6 | ControlFlowNode for module | module.py:0:0:0:0 | Module module |
+| absent.py:16:6:16:9 | ControlFlowNode for ImportExpr | file://:0:0:0:0 | Missing module xxxx |
+| absent.py:16:18:16:21 | ControlFlowNode for ImportMember | file://:0:0:0:0 | Missing module attribute xxxx.deco |
+| absent.py:18:2:18:5 | ControlFlowNode for deco | file://:0:0:0:0 | Missing module attribute xxxx.deco |
+| absent.py:18:2:18:5 | ControlFlowNode for deco() | absent.py:18:2:18:5 | Decorated Function func1 |
+| absent.py:19:1:19:12 | ControlFlowNode for FunctionExpr | absent.py:19:1:19:12 | Function func1 |
+| absent.py:22:2:22:5 | ControlFlowNode for deco | file://:0:0:0:0 | Missing module attribute xxxx.deco |
+| absent.py:22:2:22:8 | ControlFlowNode for deco()() | absent.py:22:2:22:8 | Decorated Function func2 |
+| absent.py:22:7:22:7 | ControlFlowNode for IntegerLiteral | file://:0:0:0:0 | int 0 |
+| absent.py:23:1:23:12 | ControlFlowNode for FunctionExpr | absent.py:23:1:23:12 | Function func2 |
+| absent.py:26:2:26:5 | ControlFlowNode for deco | file://:0:0:0:0 | Missing module attribute xxxx.deco |
+| absent.py:26:2:26:16 | ControlFlowNode for deco()() | absent.py:26:2:26:16 | Decorated Function func3 |
+| absent.py:27:1:27:12 | ControlFlowNode for FunctionExpr | absent.py:27:1:27:12 | Function func3 |
+| absent.py:30:2:30:5 | ControlFlowNode for deco | file://:0:0:0:0 | Missing module attribute xxxx.deco |
+| absent.py:30:2:30:11 | ControlFlowNode for deco()() | absent.py:30:2:30:11 | Decorated Function func4 |
+| absent.py:30:7:30:10 | ControlFlowNode for open | file://:0:0:0:0 | Missing module attribute xxxx.open |
+| absent.py:31:1:31:12 | ControlFlowNode for FunctionExpr | absent.py:31:1:31:12 | Function func4 |
+| absent.py:34:1:34:5 | ControlFlowNode for func1 | absent.py:18:2:18:5 | Decorated Function func1 |
+| absent.py:35:1:35:5 | ControlFlowNode for func2 | absent.py:22:2:22:8 | Decorated Function func2 |
+| absent.py:36:1:36:5 | ControlFlowNode for func3 | absent.py:26:2:26:16 | Decorated Function func3 |
+| absent.py:37:1:37:5 | ControlFlowNode for func4 | absent.py:30:2:30:11 | Decorated Function func4 |

python/ql/test/library-tests/PointsTo/absent/absent.py

@@ -12,3 +12,26 @@ open()
 import module
 
 module
+
+from xxxx import deco
+
+@deco
+def func1():
+    pass
+
+@deco(0)
+def func2():
+    pass
+
+@deco(undefined)
+def func3():
+    pass
+
+@deco(open)
+def func4():
+    pass
+
+func1
+func2
+func3
+func4

python/ql/test/library-tests/PointsTo/decorators/Values.expected

@@ -12,6 +12,7 @@
 | test.py:16:22:16:25 | ControlFlowNode for func | runtime | Unknown value |
 | test.py:16:22:16:25 | ControlFlowNode for func | test.py:30 from import | Function func3 |
 | test.py:18:21:18:24 | ControlFlowNode for args | runtime | instance of tuple |
+| test.py:20:12:20:18 | ControlFlowNode for wrapper | runtime | Decorated Function wraps2.wrapper |
 | test.py:20:12:20:18 | ControlFlowNode for wrapper | test.py:30 from import | Decorated Function wraps2.wrapper |
 | test.py:22:2:22:9 | ControlFlowNode for annotate | import | Function annotate |
 | test.py:26:2:26:7 | ControlFlowNode for wraps1 | import | Function wraps1 |